Complete Promptkit batch execution cutover
This commit is contained in:
@@ -35,9 +35,10 @@ missing_source:
|
||||
sources:
|
||||
alerts: none
|
||||
|
||||
scriptorium:
|
||||
binary: scriptorium
|
||||
promptkit:
|
||||
timeout: 2m
|
||||
local:
|
||||
concurrency_limit: 1
|
||||
|
||||
workspace:
|
||||
root: workspace
|
||||
|
||||
@@ -1,291 +0,0 @@
|
||||
// Package scriptorium adapts the external scriptorium CLI.
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
"time"
|
||||
)
|
||||
|
||||
const maxCapturedOutputBytes = 1024 * 1024
|
||||
|
||||
type CommandRunner interface {
|
||||
Run(ctx context.Context, name string, args []string, timeout time.Duration) (CommandResult, error)
|
||||
}
|
||||
|
||||
type CommandResult struct {
|
||||
Stdout []byte
|
||||
Stderr []byte
|
||||
StdoutTruncated bool
|
||||
StderrTruncated bool
|
||||
ExitCode int
|
||||
}
|
||||
|
||||
type ExecRunner struct{}
|
||||
|
||||
func (ExecRunner) Run(ctx context.Context, name string, args []string, timeout time.Duration) (CommandResult, error) {
|
||||
runCtx := ctx
|
||||
cancel := func() {}
|
||||
if timeout > 0 {
|
||||
runCtx, cancel = context.WithTimeout(ctx, timeout)
|
||||
}
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(runCtx, name, args...)
|
||||
stdout := &limitedBuffer{limit: maxCapturedOutputBytes}
|
||||
stderr := &limitedBuffer{limit: maxCapturedOutputBytes}
|
||||
cmd.Stdout = stdout
|
||||
cmd.Stderr = stderr
|
||||
err := cmd.Run()
|
||||
result := CommandResult{
|
||||
Stdout: stdout.Bytes(),
|
||||
Stderr: stderr.Bytes(),
|
||||
StdoutTruncated: stdout.Truncated(),
|
||||
StderrTruncated: stderr.Truncated(),
|
||||
ExitCode: 0,
|
||||
}
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
if runCtx.Err() != nil {
|
||||
return result, runCtx.Err()
|
||||
}
|
||||
if exitErr, ok := err.(*exec.ExitError); ok {
|
||||
result.ExitCode = exitErr.ExitCode()
|
||||
return result, nil
|
||||
}
|
||||
return result, err
|
||||
}
|
||||
|
||||
type Runner struct {
|
||||
Binary string
|
||||
ConfigPath string
|
||||
Profile string
|
||||
Timeout time.Duration
|
||||
ExtraArgs []string
|
||||
Commands CommandRunner
|
||||
}
|
||||
|
||||
type RenderRequest struct {
|
||||
PromptID string
|
||||
DataPackagePath string
|
||||
}
|
||||
|
||||
type StructuredRunRequest struct {
|
||||
PromptID string
|
||||
DataPackagePath string
|
||||
OutputPath string
|
||||
}
|
||||
|
||||
type RenderResult struct {
|
||||
Command []string `json:"command"`
|
||||
Stdout string `json:"stdout"`
|
||||
Stderr string `json:"stderr"`
|
||||
StdoutTruncated bool `json:"stdoutTruncated,omitempty"`
|
||||
StderrTruncated bool `json:"stderrTruncated,omitempty"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
}
|
||||
|
||||
type StructuredRunResult struct {
|
||||
Command []string `json:"command"`
|
||||
Stdout string `json:"stdout"`
|
||||
Stderr string `json:"stderr"`
|
||||
StdoutTruncated bool `json:"stdoutTruncated,omitempty"`
|
||||
StderrTruncated bool `json:"stderrTruncated,omitempty"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
OutputPath string `json:"outputPath"`
|
||||
}
|
||||
|
||||
func (r Runner) Render(ctx context.Context, req RenderRequest) (*RenderResult, error) {
|
||||
if req.PromptID == "" {
|
||||
return nil, fmt.Errorf("prompt id is required")
|
||||
}
|
||||
if req.DataPackagePath == "" {
|
||||
return nil, fmt.Errorf("data package path is required")
|
||||
}
|
||||
execution, err := r.execute(ctx, r.renderArgs(req))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("run scriptorium render: %w", err)
|
||||
}
|
||||
result := &RenderResult{
|
||||
Command: execution.argv(),
|
||||
Stdout: string(execution.result.Stdout),
|
||||
Stderr: string(execution.result.Stderr),
|
||||
StdoutTruncated: execution.result.StdoutTruncated,
|
||||
StderrTruncated: execution.result.StderrTruncated,
|
||||
ExitCode: execution.result.ExitCode,
|
||||
}
|
||||
if execution.result.ExitCode != 0 {
|
||||
return result, fmt.Errorf("scriptorium render exited with code %d: %s", execution.result.ExitCode, result.Stderr)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r Runner) StructuredRun(ctx context.Context, req StructuredRunRequest) (*StructuredRunResult, error) {
|
||||
result, err := r.executeRun(ctx, outputRunRequest{
|
||||
PromptID: req.PromptID,
|
||||
DataPackagePath: req.DataPackagePath,
|
||||
OutputPath: req.OutputPath,
|
||||
}, "run scriptorium structured output", "scriptorium structured run")
|
||||
if err != nil {
|
||||
if result == nil {
|
||||
return nil, err
|
||||
}
|
||||
return result.structuredRunResult(), err
|
||||
}
|
||||
return result.structuredRunResult(), nil
|
||||
}
|
||||
|
||||
func (result outputRunResult) structuredRunResult() *StructuredRunResult {
|
||||
return &StructuredRunResult{
|
||||
Command: result.Command,
|
||||
Stdout: result.Stdout,
|
||||
Stderr: result.Stderr,
|
||||
StdoutTruncated: result.StdoutTruncated,
|
||||
StderrTruncated: result.StderrTruncated,
|
||||
ExitCode: result.ExitCode,
|
||||
OutputPath: result.OutputPath,
|
||||
}
|
||||
}
|
||||
|
||||
type execution struct {
|
||||
binary string
|
||||
args []string
|
||||
result CommandResult
|
||||
}
|
||||
|
||||
type outputRunRequest struct {
|
||||
PromptID string
|
||||
DataPackagePath string
|
||||
OutputPath string
|
||||
}
|
||||
|
||||
type outputRunResult struct {
|
||||
Command []string
|
||||
Stdout string
|
||||
Stderr string
|
||||
StdoutTruncated bool
|
||||
StderrTruncated bool
|
||||
ExitCode int
|
||||
OutputPath string
|
||||
}
|
||||
|
||||
func (r Runner) executeRun(ctx context.Context, req outputRunRequest, executeContext string, exitContext string) (*outputRunResult, error) {
|
||||
if req.PromptID == "" {
|
||||
return nil, fmt.Errorf("prompt id is required")
|
||||
}
|
||||
if req.DataPackagePath == "" {
|
||||
return nil, fmt.Errorf("data package path is required")
|
||||
}
|
||||
if req.OutputPath == "" {
|
||||
return nil, fmt.Errorf("output path is required")
|
||||
}
|
||||
execution, err := r.execute(ctx, r.runArgs(req))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", executeContext, err)
|
||||
}
|
||||
result := &outputRunResult{
|
||||
Command: execution.argv(),
|
||||
Stdout: string(execution.result.Stdout),
|
||||
Stderr: string(execution.result.Stderr),
|
||||
StdoutTruncated: execution.result.StdoutTruncated,
|
||||
StderrTruncated: execution.result.StderrTruncated,
|
||||
ExitCode: execution.result.ExitCode,
|
||||
OutputPath: req.OutputPath,
|
||||
}
|
||||
if execution.result.ExitCode != 0 {
|
||||
return result, fmt.Errorf("%s exited with code %d: %s", exitContext, execution.result.ExitCode, result.Stderr)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (r Runner) execute(ctx context.Context, args []string) (execution, error) {
|
||||
binary := r.Binary
|
||||
if binary == "" {
|
||||
binary = "scriptorium"
|
||||
}
|
||||
commands := r.Commands
|
||||
if commands == nil {
|
||||
commands = ExecRunner{}
|
||||
}
|
||||
result, err := commands.Run(ctx, binary, args, r.Timeout)
|
||||
if err != nil {
|
||||
return execution{}, err
|
||||
}
|
||||
return execution{binary: binary, args: args, result: result}, nil
|
||||
}
|
||||
|
||||
func (e execution) argv() []string {
|
||||
return append([]string{e.binary}, e.args...)
|
||||
}
|
||||
|
||||
func (r Runner) renderArgs(req RenderRequest) []string {
|
||||
args := []string{"render"}
|
||||
if r.ConfigPath != "" {
|
||||
args = append(args, "--config", r.ConfigPath)
|
||||
}
|
||||
if r.Profile != "" {
|
||||
args = append(args, "--profile", r.Profile)
|
||||
}
|
||||
args = append(args,
|
||||
"--prompt", req.PromptID,
|
||||
"--input", "data_package="+req.DataPackagePath,
|
||||
"--format", "json",
|
||||
)
|
||||
args = append(args, r.ExtraArgs...)
|
||||
return args
|
||||
}
|
||||
|
||||
func (r Runner) runArgs(req outputRunRequest) []string {
|
||||
args := []string{"run"}
|
||||
if r.ConfigPath != "" {
|
||||
args = append(args, "--config", r.ConfigPath)
|
||||
}
|
||||
if r.Profile != "" {
|
||||
args = append(args, "--profile", r.Profile)
|
||||
}
|
||||
args = append(args,
|
||||
"--prompt", req.PromptID,
|
||||
"--input", "data_package="+req.DataPackagePath,
|
||||
"--out", req.OutputPath,
|
||||
)
|
||||
args = append(args, r.ExtraArgs...)
|
||||
return args
|
||||
}
|
||||
|
||||
type limitedBuffer struct {
|
||||
data []byte
|
||||
limit int
|
||||
truncated bool
|
||||
}
|
||||
|
||||
func (b *limitedBuffer) Write(p []byte) (int, error) {
|
||||
if b.limit <= 0 {
|
||||
b.truncated = true
|
||||
return len(p), nil
|
||||
}
|
||||
remaining := b.limit - len(b.data)
|
||||
if remaining <= 0 {
|
||||
b.truncated = true
|
||||
return len(p), nil
|
||||
}
|
||||
if len(p) > remaining {
|
||||
b.data = append(b.data, p[:remaining]...)
|
||||
b.truncated = true
|
||||
return len(p), nil
|
||||
}
|
||||
b.data = append(b.data, p...)
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func (b *limitedBuffer) Bytes() []byte {
|
||||
return append([]byte{}, b.data...)
|
||||
}
|
||||
|
||||
func (b *limitedBuffer) Truncated() bool {
|
||||
return b.truncated
|
||||
}
|
||||
|
||||
var _ io.Writer = (*limitedBuffer)(nil)
|
||||
@@ -1,413 +0,0 @@
|
||||
package scriptorium
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRenderConstructsCommand(t *testing.T) {
|
||||
commands := &fakeCommands{result: CommandResult{Stdout: []byte(`{"ok":true}`)}}
|
||||
runner := Runner{
|
||||
Binary: "/usr/local/bin/scriptorium",
|
||||
ConfigPath: "/etc/scriptorium.yml",
|
||||
Profile: "weather",
|
||||
Timeout: time.Minute,
|
||||
Commands: commands,
|
||||
}
|
||||
|
||||
result, err := runner.Render(context.Background(), RenderRequest{
|
||||
PromptID: "weather.daily_generated_text",
|
||||
DataPackagePath: "/tmp/data_package.yaml",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("Render() error = %v", err)
|
||||
}
|
||||
|
||||
wantArgs := []string{
|
||||
"render",
|
||||
"--config", "/etc/scriptorium.yml",
|
||||
"--profile", "weather",
|
||||
"--prompt", "weather.daily_generated_text",
|
||||
"--input", "data_package=/tmp/data_package.yaml",
|
||||
"--format", "json",
|
||||
}
|
||||
if commands.name != "/usr/local/bin/scriptorium" {
|
||||
t.Fatalf("command name = %q, want custom binary", commands.name)
|
||||
}
|
||||
if !reflect.DeepEqual(commands.args, wantArgs) {
|
||||
t.Fatalf("args = %#v, want %#v", commands.args, wantArgs)
|
||||
}
|
||||
if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) {
|
||||
t.Fatalf("result command = %#v, want full argv", result.Command)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenderReturnsResultForNonzeroExit(t *testing.T) {
|
||||
runner := Runner{
|
||||
Commands: &fakeCommands{
|
||||
result: CommandResult{
|
||||
Stderr: []byte("missing input"),
|
||||
ExitCode: 1,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := runner.Render(context.Background(), RenderRequest{
|
||||
PromptID: "weather.daily_generated_text",
|
||||
DataPackagePath: "/tmp/data_package.yaml",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("Render() error = nil, want nonzero exit error")
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("Render() result = nil, want captured result")
|
||||
}
|
||||
if result.ExitCode != 1 {
|
||||
t.Fatalf("ExitCode = %d, want 1", result.ExitCode)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing input") {
|
||||
t.Fatalf("error = %q, want stderr context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredRunConstructsCommandWithoutSchemaFlags(t *testing.T) {
|
||||
commands := &fakeCommands{result: CommandResult{
|
||||
Stdout: []byte(`{"summary":"ok"}`),
|
||||
Stderr: []byte("wrote generated text"),
|
||||
StdoutTruncated: true,
|
||||
}}
|
||||
runner := Runner{
|
||||
Binary: "/usr/local/bin/scriptorium",
|
||||
ConfigPath: "/etc/scriptorium.yml",
|
||||
Profile: "weather",
|
||||
Timeout: 30 * time.Second,
|
||||
Commands: commands,
|
||||
}
|
||||
|
||||
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
||||
PromptID: "weather.hourly_generated_text",
|
||||
DataPackagePath: "/tmp/data_package.hourly.yaml",
|
||||
OutputPath: "/tmp/generated_text_raw.hourly.json",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("StructuredRun() error = %v", err)
|
||||
}
|
||||
|
||||
wantArgs := []string{
|
||||
"run",
|
||||
"--config", "/etc/scriptorium.yml",
|
||||
"--profile", "weather",
|
||||
"--prompt", "weather.hourly_generated_text",
|
||||
"--input", "data_package=/tmp/data_package.hourly.yaml",
|
||||
"--out", "/tmp/generated_text_raw.hourly.json",
|
||||
}
|
||||
if commands.name != "/usr/local/bin/scriptorium" {
|
||||
t.Fatalf("command name = %q, want custom binary", commands.name)
|
||||
}
|
||||
if !reflect.DeepEqual(commands.args, wantArgs) {
|
||||
t.Fatalf("args = %#v, want %#v", commands.args, wantArgs)
|
||||
}
|
||||
for _, disallowed := range []string{"--format", "--schema", "--schema-path", "--json-schema"} {
|
||||
if containsArg(commands.args, disallowed) {
|
||||
t.Fatalf("args = %#v, should not include %q", commands.args, disallowed)
|
||||
}
|
||||
}
|
||||
if commands.timeout != 30*time.Second {
|
||||
t.Fatalf("timeout = %s, want 30s", commands.timeout)
|
||||
}
|
||||
if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) {
|
||||
t.Fatalf("result command = %#v, want full argv", result.Command)
|
||||
}
|
||||
if result.Stdout != `{"summary":"ok"}` || result.Stderr != "wrote generated text" || !result.StdoutTruncated {
|
||||
t.Fatalf("result = %#v, want captured output and truncation flags", result)
|
||||
}
|
||||
if result.OutputPath != "/tmp/generated_text_raw.hourly.json" {
|
||||
t.Fatalf("OutputPath = %q, want generated text raw path", result.OutputPath)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStructuredRunReturnsResultForNonzeroExit(t *testing.T) {
|
||||
runner := Runner{
|
||||
Commands: &fakeCommands{
|
||||
result: CommandResult{
|
||||
Stdout: []byte(`{"summary":"partial"}`),
|
||||
Stderr: []byte("structured output failed"),
|
||||
ExitCode: 3,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
||||
PromptID: "weather.hourly_generated_text",
|
||||
DataPackagePath: "/tmp/data_package.hourly.yaml",
|
||||
OutputPath: "/tmp/generated_text_raw.hourly.json",
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("StructuredRun() error = nil, want nonzero exit error")
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatal("StructuredRun() result = nil, want captured result")
|
||||
}
|
||||
if result.ExitCode != 3 {
|
||||
t.Fatalf("ExitCode = %d, want 3", result.ExitCode)
|
||||
}
|
||||
if result.Stdout != `{"summary":"partial"}` || result.OutputPath != "/tmp/generated_text_raw.hourly.json" {
|
||||
t.Fatalf("result = %#v, want captured result fields", result)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "structured output failed") {
|
||||
t.Fatalf("error = %q, want stderr context", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputRunsPreserveCapturedResultFields(t *testing.T) {
|
||||
type commonResult struct {
|
||||
Command []string
|
||||
Stdout string
|
||||
Stderr string
|
||||
StdoutTruncated bool
|
||||
StderrTruncated bool
|
||||
ExitCode int
|
||||
OutputPath string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(Runner) (*commonResult, error)
|
||||
}{
|
||||
{
|
||||
name: "StructuredRun",
|
||||
run: func(runner Runner) (*commonResult, error) {
|
||||
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
||||
PromptID: "weather.daily_generated_text",
|
||||
DataPackagePath: "/tmp/data_package.yaml",
|
||||
OutputPath: "/tmp/report.md",
|
||||
})
|
||||
if result == nil {
|
||||
return nil, err
|
||||
}
|
||||
return &commonResult{
|
||||
Command: result.Command,
|
||||
Stdout: result.Stdout,
|
||||
Stderr: result.Stderr,
|
||||
StdoutTruncated: result.StdoutTruncated,
|
||||
StderrTruncated: result.StderrTruncated,
|
||||
ExitCode: result.ExitCode,
|
||||
OutputPath: result.OutputPath,
|
||||
}, err
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
commands := &fakeCommands{result: CommandResult{
|
||||
Stdout: []byte("captured stdout"),
|
||||
Stderr: []byte("captured stderr"),
|
||||
StdoutTruncated: true,
|
||||
StderrTruncated: true,
|
||||
}}
|
||||
runner := Runner{
|
||||
Binary: "/usr/local/bin/scriptorium",
|
||||
ConfigPath: "/etc/scriptorium.yml",
|
||||
Profile: "weather",
|
||||
Timeout: 15 * time.Second,
|
||||
Commands: commands,
|
||||
}
|
||||
|
||||
result, err := test.run(runner)
|
||||
if err != nil {
|
||||
t.Fatalf("%s error = %v", test.name, err)
|
||||
}
|
||||
wantArgs := []string{
|
||||
"run",
|
||||
"--config", "/etc/scriptorium.yml",
|
||||
"--profile", "weather",
|
||||
"--prompt", "weather.daily_generated_text",
|
||||
"--input", "data_package=/tmp/data_package.yaml",
|
||||
"--out", "/tmp/report.md",
|
||||
}
|
||||
if !reflect.DeepEqual(commands.args, wantArgs) {
|
||||
t.Fatalf("args = %#v, want %#v", commands.args, wantArgs)
|
||||
}
|
||||
if commands.timeout != 15*time.Second {
|
||||
t.Fatalf("timeout = %s, want 15s", commands.timeout)
|
||||
}
|
||||
if !reflect.DeepEqual(result.Command, append([]string{"/usr/local/bin/scriptorium"}, wantArgs...)) {
|
||||
t.Fatalf("Command = %#v, want full argv", result.Command)
|
||||
}
|
||||
if result.Stdout != "captured stdout" || result.Stderr != "captured stderr" {
|
||||
t.Fatalf("captured output = %q/%q, want stdout/stderr", result.Stdout, result.Stderr)
|
||||
}
|
||||
if !result.StdoutTruncated || !result.StderrTruncated {
|
||||
t.Fatalf("truncation flags = %t/%t, want both true", result.StdoutTruncated, result.StderrTruncated)
|
||||
}
|
||||
if result.ExitCode != 0 || result.OutputPath != "/tmp/report.md" {
|
||||
t.Fatalf("result = %#v, want exit 0 and output path", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputRunsReturnCapturedResultForNonzeroExit(t *testing.T) {
|
||||
type commonResult struct {
|
||||
Stdout string
|
||||
Stderr string
|
||||
StderrTruncated bool
|
||||
ExitCode int
|
||||
OutputPath string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(Runner) (*commonResult, error)
|
||||
wantErr string
|
||||
}{
|
||||
{
|
||||
name: "StructuredRun",
|
||||
run: func(runner Runner) (*commonResult, error) {
|
||||
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
||||
PromptID: "weather.daily_generated_text",
|
||||
DataPackagePath: "/tmp/data_package.yaml",
|
||||
OutputPath: "/tmp/report.md",
|
||||
})
|
||||
if result == nil {
|
||||
return nil, err
|
||||
}
|
||||
return &commonResult{
|
||||
Stdout: result.Stdout,
|
||||
Stderr: result.Stderr,
|
||||
StderrTruncated: result.StderrTruncated,
|
||||
ExitCode: result.ExitCode,
|
||||
OutputPath: result.OutputPath,
|
||||
}, err
|
||||
},
|
||||
wantErr: "scriptorium structured run exited with code 7: captured stderr",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
runner := Runner{
|
||||
Commands: &fakeCommands{result: CommandResult{
|
||||
Stdout: []byte("captured stdout"),
|
||||
Stderr: []byte("captured stderr"),
|
||||
StderrTruncated: true,
|
||||
ExitCode: 7,
|
||||
}},
|
||||
}
|
||||
|
||||
result, err := test.run(runner)
|
||||
if err == nil {
|
||||
t.Fatalf("%s error = nil, want nonzero exit error", test.name)
|
||||
}
|
||||
if result == nil {
|
||||
t.Fatalf("%s result = nil, want captured result", test.name)
|
||||
}
|
||||
if err.Error() != test.wantErr {
|
||||
t.Fatalf("%s error = %q, want %q", test.name, err.Error(), test.wantErr)
|
||||
}
|
||||
if result.Stdout != "captured stdout" || result.Stderr != "captured stderr" || !result.StderrTruncated {
|
||||
t.Fatalf("captured result = %#v, want stdout/stderr/truncation", result)
|
||||
}
|
||||
if result.ExitCode != 7 || result.OutputPath != "/tmp/report.md" {
|
||||
t.Fatalf("result = %#v, want exit 7 and output path", result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputRunsValidateRequiredFieldsBeforeExecution(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
run func(Runner, string, string, string) error
|
||||
}{
|
||||
{
|
||||
name: "StructuredRun",
|
||||
run: func(runner Runner, promptID string, dataPackagePath string, outputPath string) error {
|
||||
result, err := runner.StructuredRun(context.Background(), StructuredRunRequest{
|
||||
PromptID: promptID,
|
||||
DataPackagePath: dataPackagePath,
|
||||
OutputPath: outputPath,
|
||||
})
|
||||
if result != nil {
|
||||
return fmt.Errorf("result = %#v, want nil", result)
|
||||
}
|
||||
return err
|
||||
},
|
||||
},
|
||||
}
|
||||
cases := []struct {
|
||||
name string
|
||||
promptID string
|
||||
dataPackagePath string
|
||||
outputPath string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "prompt id",
|
||||
dataPackagePath: "/tmp/data_package.yaml",
|
||||
outputPath: "/tmp/report.md",
|
||||
want: "prompt id is required",
|
||||
},
|
||||
{
|
||||
name: "data package path",
|
||||
promptID: "weather.daily_generated_text",
|
||||
outputPath: "/tmp/report.md",
|
||||
want: "data package path is required",
|
||||
},
|
||||
{
|
||||
name: "output path",
|
||||
promptID: "weather.daily_generated_text",
|
||||
dataPackagePath: "/tmp/data_package.yaml",
|
||||
want: "output path is required",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
commands := &fakeCommands{}
|
||||
err := test.run(Runner{Commands: commands}, tc.promptID, tc.dataPackagePath, tc.outputPath)
|
||||
if err == nil {
|
||||
t.Fatalf("%s error = nil, want validation error", test.name)
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.want) {
|
||||
t.Fatalf("%s error = %v, want %q", test.name, err, tc.want)
|
||||
}
|
||||
if commands.calls != 0 {
|
||||
t.Fatalf("commands calls = %d, want no subprocess execution", commands.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type fakeCommands struct {
|
||||
name string
|
||||
args []string
|
||||
timeout time.Duration
|
||||
result CommandResult
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeCommands) Run(_ context.Context, name string, args []string, timeout time.Duration) (CommandResult, error) {
|
||||
f.calls++
|
||||
f.name = name
|
||||
f.args = append([]string{}, args...)
|
||||
f.timeout = timeout
|
||||
return f.result, f.err
|
||||
}
|
||||
|
||||
func containsArg(args []string, want string) bool {
|
||||
for _, arg := range args {
|
||||
if arg == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -3,13 +3,11 @@ package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
distributoradapter "gitea.maximumdirect.net/eric/weatherreporter/internal/adapters/distributor"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/adapters/scriptorium"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/briefing"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/changes"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
@@ -17,7 +15,6 @@ import (
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/facts"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/fileutil"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/forecast"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/module"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptinput"
|
||||
@@ -57,14 +54,15 @@ type GenerateRequest struct {
|
||||
}
|
||||
|
||||
type BatchRequest struct {
|
||||
Config config.Config
|
||||
Batch BatchKind
|
||||
Now time.Time
|
||||
OutputDir string
|
||||
Collector Collector
|
||||
Renderer Renderer
|
||||
Store state.Store
|
||||
Notifier Notifier
|
||||
Config config.Config
|
||||
Batch BatchKind
|
||||
Now time.Time
|
||||
OutputDir string
|
||||
LLMDebugDir string
|
||||
Collector Collector
|
||||
Executor promptexec.Executor
|
||||
Store state.Store
|
||||
Notifier Notifier
|
||||
}
|
||||
|
||||
type FetchBundleRequest struct {
|
||||
@@ -82,17 +80,6 @@ type ReportFacts struct {
|
||||
Derived facts.DerivedFacts
|
||||
}
|
||||
|
||||
type ReportRequest struct {
|
||||
Config config.Config
|
||||
Resolved report.Resolved
|
||||
OutputPath string
|
||||
Collection collect.Result
|
||||
Renderer Renderer
|
||||
Store state.Store
|
||||
Notifier Notifier
|
||||
noNotify bool
|
||||
}
|
||||
|
||||
type ReportResult struct {
|
||||
ModuleSnapshot module.Snapshot
|
||||
ModuleSnapshotPath string
|
||||
@@ -101,7 +88,6 @@ type ReportResult struct {
|
||||
PreparationPath string
|
||||
ExecutionPath string
|
||||
LLMDebugPath string
|
||||
PreflightPath string
|
||||
ReportPath string
|
||||
OutputPath string
|
||||
NotificationPath string
|
||||
@@ -202,11 +188,6 @@ func batchReportFailures(result *BatchResult) int {
|
||||
return failures
|
||||
}
|
||||
|
||||
type Renderer interface {
|
||||
Render(context.Context, scriptorium.RenderRequest) (*scriptorium.RenderResult, error)
|
||||
StructuredRun(context.Context, scriptorium.StructuredRunRequest) (*scriptorium.StructuredRunResult, error)
|
||||
}
|
||||
|
||||
type Collector interface {
|
||||
Run(context.Context, collect.Request) (*collect.Result, error)
|
||||
}
|
||||
@@ -324,6 +305,22 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
||||
if _, err := report.BatchForCommandName(string(req.Batch)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
debugWriter, err := state.NewPromptDebugWriter(req.LLMDebugDir)
|
||||
if err != nil {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||
}
|
||||
candidates, err := batchInspectionCandidates(req, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
inspections, err := InspectPromptExecutions(ctx, PromptExecutionsInspectionRequest{
|
||||
Resolved: candidates,
|
||||
Executor: req.Executor,
|
||||
Promptkit: req.Config.Promptkit,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -348,49 +345,35 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
||||
item := batchReportResult(planned)
|
||||
if paths, err := store.Paths(resolved); err == nil {
|
||||
item.DataPackagePath = paths.DataPackage
|
||||
item.PreparationPath = paths.Preflight
|
||||
item.PreparationPath = paths.Preparation
|
||||
item.ExecutionPath = paths.Execution
|
||||
item.ReportPath = paths.RenderedReport
|
||||
item.MetadataPath = paths.Metadata
|
||||
}
|
||||
outputPath := plannedBatchOutputPath(req.OutputDir, planned)
|
||||
reportResult, err := generateLegacyBatchReport(ctx, ReportRequest{
|
||||
Config: req.Config,
|
||||
Resolved: resolved,
|
||||
OutputPath: outputPath,
|
||||
Collection: *collection,
|
||||
Renderer: req.Renderer,
|
||||
Store: store,
|
||||
Notifier: req.Notifier,
|
||||
noNotify: true,
|
||||
reportResult, err := generatePromptReport(ctx, promptReportRequest{
|
||||
GenerateRequest: GenerateRequest{
|
||||
Config: req.Config,
|
||||
OutputPath: outputPath,
|
||||
Notifier: req.Notifier,
|
||||
Executor: req.Executor,
|
||||
Store: store,
|
||||
},
|
||||
Resolved: resolved,
|
||||
Collection: *collection,
|
||||
Inspection: inspections[resolved.Definition.ID],
|
||||
DebugWriter: debugWriter,
|
||||
noNotify: true,
|
||||
})
|
||||
if reportResult != nil {
|
||||
copyBatchReportPaths(&item, reportResult)
|
||||
}
|
||||
if err != nil {
|
||||
item.Status = "failed"
|
||||
item.Error = err.Error()
|
||||
var notificationErr *NotificationError
|
||||
if errors.As(err, ¬ificationErr) {
|
||||
item.NotificationStatus = "failed"
|
||||
item.NotificationError = notificationErr.Error()
|
||||
item.NotificationPipelineID = notificationErr.Request.PipelineID
|
||||
if paths, pathErr := store.Paths(resolved); pathErr == nil {
|
||||
item.NotificationPath = paths.Notification
|
||||
}
|
||||
}
|
||||
result.Failed++
|
||||
} else {
|
||||
item.Status = "succeeded"
|
||||
item.DataPackagePath = reportResult.DataPackagePath
|
||||
item.PreparationPath = reportResult.PreparationPath
|
||||
item.ExecutionPath = reportResult.ExecutionPath
|
||||
item.LLMDebugPath = reportResult.LLMDebugPath
|
||||
item.ReportPath = reportResult.ReportPath
|
||||
item.OutputPath = reportResult.OutputPath
|
||||
item.MetadataPath = reportResult.MetadataPath
|
||||
item.NotificationPath = reportResult.NotificationPath
|
||||
if reportResult.Notification != nil {
|
||||
item.NotificationStatus = reportResult.Notification.Status
|
||||
item.NotificationRunID = reportResult.Notification.RunID
|
||||
item.NotificationPipelineID = reportResult.Notification.PipelineID
|
||||
}
|
||||
result.Succeeded++
|
||||
}
|
||||
result.Reports = append(result.Reports, item)
|
||||
@@ -409,6 +392,51 @@ func RunBatchDetailed(ctx context.Context, req BatchRequest) (*BatchResult, erro
|
||||
return nil, fmt.Errorf("run is not implemented")
|
||||
}
|
||||
|
||||
func copyBatchReportPaths(item *BatchReportResult, result *ReportResult) {
|
||||
item.DataPackagePath = result.DataPackagePath
|
||||
item.PreparationPath = result.PreparationPath
|
||||
item.ExecutionPath = result.ExecutionPath
|
||||
item.LLMDebugPath = result.LLMDebugPath
|
||||
item.ReportPath = result.ReportPath
|
||||
item.OutputPath = result.OutputPath
|
||||
item.MetadataPath = result.MetadataPath
|
||||
item.NotificationPath = result.NotificationPath
|
||||
if result.Notification != nil {
|
||||
item.NotificationStatus = result.Notification.Status
|
||||
item.NotificationRunID = result.Notification.RunID
|
||||
item.NotificationPipelineID = result.Notification.PipelineID
|
||||
}
|
||||
}
|
||||
|
||||
func batchInspectionCandidates(req BatchRequest, now time.Time) ([]report.Resolved, error) {
|
||||
location, err := timeutil.LoadLocation(req.Config.WeatherAPI.Timezone)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registry, err := reportRegistry(req.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ids := []report.ID{report.Tomorrow, report.Daily}
|
||||
if req.Batch == BatchMorning {
|
||||
ids = []report.ID{report.Today, report.Tomorrow, report.Daily}
|
||||
}
|
||||
date := timeutil.LocalDate(now, location).AddDate(0, 0, 2)
|
||||
candidates := make([]report.Resolved, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
resolveReq := report.ResolveRequest{Now: now, Location: location}
|
||||
if id == report.Daily {
|
||||
resolveReq.Date = date
|
||||
}
|
||||
resolved, err := registry.Resolve(id, resolveReq)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
candidates = append(candidates, resolved)
|
||||
}
|
||||
return candidates, nil
|
||||
}
|
||||
|
||||
func batchReportResult(planned plannedBatchReport) BatchReportResult {
|
||||
resolved := planned.Resolved
|
||||
metadata := resolved.Metadata()
|
||||
@@ -507,315 +535,6 @@ func FetchAndSaveBundle(ctx context.Context, req FetchBundleRequest) (*weatherda
|
||||
return bundle, nil
|
||||
}
|
||||
|
||||
// generateLegacyBatchReport is the temporary Scriptorium implementation used
|
||||
// only by batch commands while their Promptkit migration is deferred.
|
||||
func generateLegacyBatchReport(ctx context.Context, req ReportRequest) (*ReportResult, error) {
|
||||
bundle := req.Collection.Bundle
|
||||
if bundle == nil {
|
||||
return nil, fmt.Errorf("collected weather bundle is required")
|
||||
}
|
||||
|
||||
store := req.Store
|
||||
if store == nil {
|
||||
defaultStore, err := defaultStore(req.Config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
store = defaultStore
|
||||
}
|
||||
paths, err := store.Paths(req.Resolved)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
priorSnapshot, err := store.FindPriorSnapshot(ctx, req.Resolved)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
reportFacts, err := BuildReportFacts(ModuleSnapshotRequest{
|
||||
Config: req.Config,
|
||||
Resolved: req.Resolved,
|
||||
}, bundle)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
moduleSnapshot, err := BuildModuleSnapshotFromFacts(ModuleSnapshotRequest{
|
||||
Config: req.Config,
|
||||
Resolved: req.Resolved,
|
||||
}, reportFacts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
moduleSnapshotPath, err := store.SaveModuleSnapshot(ctx, req.Resolved, moduleSnapshot)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
recentChanges, err := recentChanges(ctx, store, priorSnapshot, req.Resolved.Definition.ID, moduleSnapshot, req.Config.RecentChange)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
briefingMetadata := briefing.BuildMetadata(briefingBuildContext(req.Config, req.Resolved, reportFacts.Collected))
|
||||
metadata := state.BuildMetadataFromBriefingMetadata(req.Resolved, briefingMetadata, state.ArtifactPaths{
|
||||
ModuleSnapshot: moduleSnapshotPath,
|
||||
Metadata: paths.Metadata,
|
||||
DataPackage: paths.DataPackage,
|
||||
Preflight: paths.Preflight,
|
||||
RenderedReport: paths.RenderedReport,
|
||||
GeneratedTextRaw: paths.GeneratedTextRaw,
|
||||
GeneratedTextResult: paths.GeneratedTextResult,
|
||||
GeneratedText: paths.GeneratedText,
|
||||
RenderContext: paths.RenderContext,
|
||||
})
|
||||
dataPackage, err := promptinput.Build(promptinput.BuildRequest{
|
||||
Metadata: promptMetadata(metadata),
|
||||
Modules: moduleSnapshot,
|
||||
RecentChanges: recentChanges,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dataPackagePath, err := store.SaveDataPackage(ctx, req.Resolved, dataPackage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata.DataPackagePath = dataPackagePath
|
||||
|
||||
renderer := req.Renderer
|
||||
if renderer == nil {
|
||||
renderer = scriptorium.Runner{
|
||||
Binary: req.Config.Scriptorium.Binary,
|
||||
ConfigPath: req.Config.Scriptorium.ConfigPath,
|
||||
Profile: req.Config.Scriptorium.Profile,
|
||||
Timeout: req.Config.Scriptorium.Timeout,
|
||||
ExtraArgs: req.Config.Scriptorium.ExtraArgs,
|
||||
}
|
||||
}
|
||||
renderResult, renderErr := renderer.Render(ctx, scriptorium.RenderRequest{
|
||||
PromptID: req.Resolved.Definition.PromptID,
|
||||
DataPackagePath: dataPackagePath,
|
||||
})
|
||||
|
||||
preflightPath := paths.Preflight
|
||||
if renderResult != nil {
|
||||
var err error
|
||||
preflightPath, err = store.SavePreflight(ctx, req.Resolved, preflightArtifact(renderResult))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
metadata.PreflightPath = preflightPath
|
||||
metadataPath, metadataErr := store.SaveMetadata(ctx, metadata)
|
||||
if metadataErr != nil {
|
||||
return nil, metadataErr
|
||||
}
|
||||
if renderErr != nil {
|
||||
return nil, generatedReportError(req.Resolved, metadata.RunID, "render preflight", renderErr)
|
||||
}
|
||||
|
||||
return generateTextTemplateReport(ctx, generatedReportRequest{
|
||||
ReportRequest: req,
|
||||
store: store,
|
||||
paths: paths,
|
||||
moduleSnapshot: moduleSnapshot,
|
||||
moduleSnapshotPath: moduleSnapshotPath,
|
||||
reportFacts: reportFacts,
|
||||
dataPackage: dataPackage,
|
||||
dataPackagePath: dataPackagePath,
|
||||
briefingMetadata: briefingMetadata,
|
||||
metadata: metadata,
|
||||
metadataPath: metadataPath,
|
||||
preflightPath: preflightPath,
|
||||
priorSnapshot: priorSnapshot,
|
||||
recentChanges: recentChanges,
|
||||
renderResult: renderResult,
|
||||
renderer: renderer,
|
||||
})
|
||||
}
|
||||
|
||||
type generatedReportRequest struct {
|
||||
ReportRequest
|
||||
store state.Store
|
||||
paths state.ArtifactPaths
|
||||
moduleSnapshot module.Snapshot
|
||||
moduleSnapshotPath string
|
||||
reportFacts ReportFacts
|
||||
dataPackage promptinput.Package
|
||||
dataPackagePath string
|
||||
briefingMetadata briefing.Metadata
|
||||
metadata state.Metadata
|
||||
metadataPath string
|
||||
preflightPath string
|
||||
priorSnapshot *state.PriorSnapshot
|
||||
recentChanges []changes.Change
|
||||
renderResult *scriptorium.RenderResult
|
||||
renderer Renderer
|
||||
}
|
||||
|
||||
func generateTextTemplateReport(ctx context.Context, req generatedReportRequest) (*ReportResult, error) {
|
||||
handler, err := generatedtext.LookupDefinition(req.Resolved.Definition)
|
||||
if err != nil {
|
||||
return nil, generatedReportError(req.Resolved, req.metadata.RunID, "lookup generated text catalog", err)
|
||||
}
|
||||
|
||||
structuredResult, runErr := req.renderer.StructuredRun(ctx, scriptorium.StructuredRunRequest{
|
||||
PromptID: req.Resolved.Definition.PromptID,
|
||||
DataPackagePath: req.dataPackagePath,
|
||||
OutputPath: req.paths.GeneratedTextRaw,
|
||||
})
|
||||
generatedTextResultPath := req.paths.GeneratedTextResult
|
||||
if structuredResult != nil {
|
||||
var err error
|
||||
generatedTextResultPath, err = req.store.SaveGeneratedTextResult(ctx, req.Resolved, structuredResult)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.metadata.GeneratedTextResultPath = generatedTextResultPath
|
||||
req.metadataPath, err = req.store.SaveMetadata(ctx, req.metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if runErr != nil {
|
||||
return nil, generatedReportError(req.Resolved, req.metadata.RunID, "structured generated text", runErr)
|
||||
}
|
||||
|
||||
rawGeneratedText, err := req.store.LoadGeneratedText(ctx, req.paths.GeneratedTextRaw)
|
||||
if err != nil {
|
||||
return nil, generatedReportError(req.Resolved, req.metadata.RunID, "load raw generated text", err)
|
||||
}
|
||||
generatedText, normalizedGeneratedText, err := handler.Validate(rawGeneratedText)
|
||||
if err != nil {
|
||||
return nil, generatedReportError(req.Resolved, req.metadata.RunID, "validate generated text", err)
|
||||
}
|
||||
generatedTextPath, err := req.store.SaveGeneratedText(ctx, req.Resolved, normalizedGeneratedText)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.metadata.GeneratedTextPath = generatedTextPath
|
||||
req.metadataPath, err = req.store.SaveMetadata(ctx, req.metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
renderContext, err := handler.BuildRenderContext(req.briefingMetadata, req.moduleSnapshot, req.reportFacts.Collected, req.reportFacts.Derived, generatedText)
|
||||
if err != nil {
|
||||
return nil, generatedReportError(req.Resolved, req.metadata.RunID, "build render context", err)
|
||||
}
|
||||
renderContextPath, err := req.store.SaveRenderContext(ctx, req.Resolved, renderContext)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.metadata.RenderContextPath = renderContextPath
|
||||
req.metadataPath, err = req.store.SaveMetadata(ctx, req.metadata)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
rendered, err := handler.Render(renderContext)
|
||||
if err != nil {
|
||||
return nil, generatedReportError(req.Resolved, req.metadata.RunID, "render template", err)
|
||||
}
|
||||
reportPath, err := req.store.PrepareRenderedReport(ctx, req.Resolved)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := fileutil.WriteFileAtomic(reportPath, rendered); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
finalized, err := finalizeRenderedReport(ctx, finalizeRenderedReportRequest{
|
||||
Config: req.Config,
|
||||
Store: req.store,
|
||||
Resolved: req.Resolved,
|
||||
Metadata: req.metadata,
|
||||
ManagedReportPath: reportPath,
|
||||
OutputPath: req.OutputPath,
|
||||
Notifier: req.Notifier,
|
||||
noNotify: req.noNotify,
|
||||
})
|
||||
if err != nil {
|
||||
if finalizeResultEmpty(finalized) {
|
||||
return nil, err
|
||||
}
|
||||
return renderedReportResult(reportResultRequest{
|
||||
moduleSnapshot: req.moduleSnapshot,
|
||||
moduleSnapshotPath: req.moduleSnapshotPath,
|
||||
dataPackage: req.dataPackage,
|
||||
dataPackagePath: req.dataPackagePath,
|
||||
preflightPath: req.preflightPath,
|
||||
reportPath: reportPath,
|
||||
finalized: finalized,
|
||||
priorSnapshot: req.priorSnapshot,
|
||||
recentChanges: req.recentChanges,
|
||||
generatedTextRawPath: req.paths.GeneratedTextRaw,
|
||||
generatedTextPath: generatedTextPath,
|
||||
renderContextPath: renderContextPath,
|
||||
}), err
|
||||
}
|
||||
|
||||
return renderedReportResult(reportResultRequest{
|
||||
moduleSnapshot: req.moduleSnapshot,
|
||||
moduleSnapshotPath: req.moduleSnapshotPath,
|
||||
dataPackage: req.dataPackage,
|
||||
dataPackagePath: req.dataPackagePath,
|
||||
preflightPath: req.preflightPath,
|
||||
reportPath: reportPath,
|
||||
finalized: finalized,
|
||||
priorSnapshot: req.priorSnapshot,
|
||||
recentChanges: req.recentChanges,
|
||||
generatedTextRawPath: req.paths.GeneratedTextRaw,
|
||||
generatedTextPath: generatedTextPath,
|
||||
renderContextPath: renderContextPath,
|
||||
}), nil
|
||||
}
|
||||
|
||||
func finalizeResultEmpty(result finalizeRenderedReportResult) bool {
|
||||
return result.OutputPath == "" &&
|
||||
result.NotificationPath == "" &&
|
||||
result.MetadataPath == "" &&
|
||||
result.Metadata.RunID == "" &&
|
||||
result.Notification == nil
|
||||
}
|
||||
|
||||
type reportResultRequest struct {
|
||||
moduleSnapshot module.Snapshot
|
||||
moduleSnapshotPath string
|
||||
dataPackage promptinput.Package
|
||||
dataPackagePath string
|
||||
preflightPath string
|
||||
reportPath string
|
||||
finalized finalizeRenderedReportResult
|
||||
priorSnapshot *state.PriorSnapshot
|
||||
recentChanges []changes.Change
|
||||
generatedTextRawPath string
|
||||
generatedTextPath string
|
||||
renderContextPath string
|
||||
}
|
||||
|
||||
func renderedReportResult(req reportResultRequest) *ReportResult {
|
||||
return &ReportResult{
|
||||
ModuleSnapshot: req.moduleSnapshot,
|
||||
ModuleSnapshotPath: req.moduleSnapshotPath,
|
||||
DataPackage: req.dataPackage,
|
||||
DataPackagePath: req.dataPackagePath,
|
||||
PreparationPath: req.preflightPath,
|
||||
ExecutionPath: req.finalized.Metadata.GeneratedTextResultPath,
|
||||
PreflightPath: req.preflightPath,
|
||||
ReportPath: req.reportPath,
|
||||
OutputPath: req.finalized.OutputPath,
|
||||
NotificationPath: req.finalized.NotificationPath,
|
||||
Metadata: req.finalized.Metadata,
|
||||
MetadataPath: req.finalized.MetadataPath,
|
||||
PriorSnapshot: req.priorSnapshot,
|
||||
RecentChanges: req.recentChanges,
|
||||
GeneratedTextRawPath: req.generatedTextRawPath,
|
||||
GeneratedTextPath: req.generatedTextPath,
|
||||
RenderContextPath: req.renderContextPath,
|
||||
Notification: req.finalized.Notification,
|
||||
}
|
||||
}
|
||||
|
||||
type finalizeRenderedReportRequest struct {
|
||||
Config config.Config
|
||||
Store state.Store
|
||||
@@ -1292,20 +1011,6 @@ func recentChanges(ctx context.Context, store state.Store, priorSnapshot *state.
|
||||
}
|
||||
}
|
||||
|
||||
func preflightArtifact(result *scriptorium.RenderResult) state.PreflightArtifact {
|
||||
if result == nil {
|
||||
return state.PreflightArtifact{}
|
||||
}
|
||||
return state.PreflightArtifact{
|
||||
Command: append([]string(nil), result.Command...),
|
||||
Stdout: result.Stdout,
|
||||
Stderr: result.Stderr,
|
||||
StdoutTruncated: result.StdoutTruncated,
|
||||
StderrTruncated: result.StderrTruncated,
|
||||
ExitCode: result.ExitCode,
|
||||
}
|
||||
}
|
||||
|
||||
func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
48
internal/app/batch_execution_test.go
Normal file
48
internal/app/batch_execution_test.go
Normal file
@@ -0,0 +1,48 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
)
|
||||
|
||||
func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
|
||||
cfg := config.Defaults()
|
||||
cfg.Workspace.Root = t.TempDir()
|
||||
now := mustParse("2026-05-29T08:00:00-05:00")
|
||||
req := BatchRequest{Config: cfg, Batch: BatchMorning, Now: now}
|
||||
candidates, err := batchInspectionCandidates(req, now)
|
||||
if err != nil {
|
||||
t.Fatalf("batchInspectionCandidates() error = %v", err)
|
||||
}
|
||||
executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{
|
||||
"default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"},
|
||||
}, prompts: map[string]promptexec.PromptInspection{}}
|
||||
for _, candidate := range candidates {
|
||||
executor.prompts[candidate.Definition.PromptID] = validPromptInspection(candidate.Definition)
|
||||
}
|
||||
collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) {
|
||||
return nil, errors.New("collection reached")
|
||||
})
|
||||
req.Executor = executor
|
||||
req.Collector = collector
|
||||
_, err = RunBatchDetailed(context.Background(), req)
|
||||
if err == nil || err.Error() != "collection reached" {
|
||||
t.Fatalf("RunBatchDetailed() error = %v, want collection error", err)
|
||||
}
|
||||
if len(executor.promptRequests) != 3 || len(executor.profileRequests) != 1 {
|
||||
t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
|
||||
}
|
||||
}
|
||||
|
||||
type collectorFunc func(context.Context, collect.Request) (*collect.Result, error)
|
||||
|
||||
func (f collectorFunc) Run(ctx context.Context, req collect.Request) (*collect.Result, error) {
|
||||
return f(ctx, req)
|
||||
}
|
||||
|
||||
var _ Collector = collectorFunc(nil)
|
||||
@@ -21,6 +21,7 @@ type promptReportRequest struct {
|
||||
Collection collect.Result
|
||||
Inspection PromptInspectionResult
|
||||
DebugWriter *state.PromptDebugWriter
|
||||
noNotify bool
|
||||
}
|
||||
|
||||
func generatePromptReport(ctx context.Context, req promptReportRequest) (*ReportResult, error) {
|
||||
@@ -315,7 +316,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report
|
||||
result.ReportPath = reportPath
|
||||
finalized, err := finalizeRenderedReport(ctx, finalizeRenderedReportRequest{
|
||||
Config: req.Config, Store: store, Resolved: req.Resolved, Metadata: metadata,
|
||||
ManagedReportPath: reportPath, OutputPath: req.OutputPath, Notifier: req.Notifier,
|
||||
ManagedReportPath: reportPath, OutputPath: req.OutputPath, Notifier: req.Notifier, noNotify: req.noNotify,
|
||||
})
|
||||
result.OutputPath, result.NotificationPath = finalized.OutputPath, finalized.NotificationPath
|
||||
result.Metadata, result.MetadataPath, result.Notification = finalized.Metadata, finalized.MetadataPath, finalized.Notification
|
||||
|
||||
@@ -30,64 +30,100 @@ type PromptInspectionResult struct {
|
||||
ModelName string
|
||||
}
|
||||
|
||||
// PromptExecutionsInspectionRequest validates all prompt/profile combinations
|
||||
// needed by a batch before collection begins.
|
||||
type PromptExecutionsInspectionRequest struct {
|
||||
Resolved []report.Resolved
|
||||
Executor promptexec.Executor
|
||||
Promptkit config.PromptkitConfig
|
||||
LookupEnv func(string) (string, bool)
|
||||
}
|
||||
|
||||
// InspectPromptExecution validates the exact prompt and profile needed for a
|
||||
// report before collection, execution, or durable writes begin.
|
||||
func InspectPromptExecution(ctx context.Context, req PromptInspectionRequest) (PromptInspectionResult, error) {
|
||||
results, err := InspectPromptExecutions(ctx, PromptExecutionsInspectionRequest{
|
||||
Resolved: []report.Resolved{req.Resolved},
|
||||
Executor: req.Executor,
|
||||
Promptkit: req.Promptkit,
|
||||
LookupEnv: req.LookupEnv,
|
||||
})
|
||||
if err != nil {
|
||||
return PromptInspectionResult{}, err
|
||||
}
|
||||
return results[req.Resolved.Definition.ID], nil
|
||||
}
|
||||
|
||||
// InspectPromptExecutions validates exact prompt contracts and their unique
|
||||
// effective profiles. It performs no collection, execution, or durable write.
|
||||
func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspectionRequest) (map[report.ID]PromptInspectionResult, error) {
|
||||
if req.Executor == nil {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil)
|
||||
}
|
||||
definition := req.Resolved.Definition
|
||||
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
|
||||
results := make(map[report.ID]PromptInspectionResult, len(req.Resolved))
|
||||
profiles := map[string]promptexec.ProfileInspection{}
|
||||
for _, resolved := range req.Resolved {
|
||||
definition := resolved.Definition
|
||||
if strings.TrimSpace(definition.PromptID) == "" || strings.TrimSpace(definition.PromptVersion) == "" {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "report prompt identity is incomplete", nil)
|
||||
}
|
||||
inspection, err := req.Executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
|
||||
if err != nil {
|
||||
return nil, promptInspectionError("prompt inspection failed", err)
|
||||
}
|
||||
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
|
||||
}
|
||||
if !validPromptInput(inspection.Inputs) {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
|
||||
}
|
||||
if !validPromptOutput(definition, inspection.Output) {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
|
||||
}
|
||||
profileID := req.Promptkit.Profile
|
||||
if profileID == "" {
|
||||
profileID = inspection.DefaultProfileID
|
||||
}
|
||||
if strings.TrimSpace(profileID) == "" {
|
||||
return nil, promptexec.NewError(promptexec.InvalidConfiguration, "prompt has no execution profile", nil)
|
||||
}
|
||||
profile, ok := profiles[profileID]
|
||||
if !ok {
|
||||
profile, err = inspectPromptProfile(ctx, req.Executor, profileID, req.LookupEnv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
profiles[profileID] = profile
|
||||
}
|
||||
results[definition.ID] = PromptInspectionResult{
|
||||
PromptID: inspection.PromptID, PromptVersion: inspection.PromptVersion, PromptHash: inspection.PromptHash,
|
||||
ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName,
|
||||
}
|
||||
}
|
||||
inspection, err := req.Executor.InspectPrompt(ctx, definition.PromptID, definition.PromptVersion)
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func inspectPromptProfile(ctx context.Context, executor promptexec.Executor, profileID string, lookupEnv func(string) (string, bool)) (promptexec.ProfileInspection, error) {
|
||||
profile, err := executor.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
return PromptInspectionResult{}, promptInspectionError("prompt inspection failed", err)
|
||||
}
|
||||
if inspection.PromptID != definition.PromptID || inspection.PromptVersion != definition.PromptVersion {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt inspection did not return the requested prompt version", nil)
|
||||
}
|
||||
if !validPromptInput(inspection.Inputs) {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare exactly one required application/yaml data_package input", nil)
|
||||
}
|
||||
if !validPromptOutput(definition, inspection.Output) {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt must declare the report JSON Schema output contract", nil)
|
||||
}
|
||||
profileID := req.Promptkit.Profile
|
||||
if profileID == "" {
|
||||
profileID = inspection.DefaultProfileID
|
||||
}
|
||||
if strings.TrimSpace(profileID) == "" {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt has no execution profile", nil)
|
||||
}
|
||||
profile, err := req.Executor.InspectProfile(ctx, profileID)
|
||||
if err != nil {
|
||||
return PromptInspectionResult{}, promptInspectionError("profile inspection failed", err)
|
||||
return promptexec.ProfileInspection{}, promptInspectionError("profile inspection failed", err)
|
||||
}
|
||||
if profile.ProfileID != profileID {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "profile inspection did not return the selected profile", nil)
|
||||
return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.InvalidConfiguration, "profile inspection did not return the selected profile", nil)
|
||||
}
|
||||
if profile.CredentialRequired {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.MissingCredential, "selected profile requires an unsupported direct API key", nil)
|
||||
return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.MissingCredential, "selected profile requires an unsupported direct API key", nil)
|
||||
}
|
||||
if strings.TrimSpace(profile.APIKeyEnv) != "" {
|
||||
lookupEnv := req.LookupEnv
|
||||
if lookupEnv == nil {
|
||||
lookupEnv = os.LookupEnv
|
||||
}
|
||||
value, present := lookupEnv(profile.APIKeyEnv)
|
||||
if !present || strings.TrimSpace(value) == "" {
|
||||
return PromptInspectionResult{}, promptexec.NewError(promptexec.MissingCredential, "selected profile credential is unavailable", nil)
|
||||
return promptexec.ProfileInspection{}, promptexec.NewError(promptexec.MissingCredential, "selected profile credential is unavailable", nil)
|
||||
}
|
||||
}
|
||||
return PromptInspectionResult{
|
||||
PromptID: inspection.PromptID,
|
||||
PromptVersion: inspection.PromptVersion,
|
||||
PromptHash: inspection.PromptHash,
|
||||
ProfileID: profile.ProfileID,
|
||||
BackendID: profile.BackendID,
|
||||
ModelName: profile.ModelName,
|
||||
}, nil
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func validPromptInput(inputs []promptexec.InputDefinition) bool {
|
||||
|
||||
@@ -107,6 +107,30 @@ func TestInspectPromptExecutionReturnsSafeInspectionError(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
|
||||
first := inspectionResolved(t)
|
||||
second := first
|
||||
second.Definition.ID = report.Today
|
||||
second.Definition.PromptID = "weather.today"
|
||||
executor := &inspectionExecutor{
|
||||
prompt: validPromptInspection(first.Definition),
|
||||
profiles: map[string]promptexec.ProfileInspection{
|
||||
"default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"},
|
||||
},
|
||||
}
|
||||
executor.prompts = map[string]promptexec.PromptInspection{
|
||||
first.Definition.PromptID: validPromptInspection(first.Definition),
|
||||
second.Definition.PromptID: validPromptInspection(second.Definition),
|
||||
}
|
||||
results, err := InspectPromptExecutions(context.Background(), PromptExecutionsInspectionRequest{Resolved: []report.Resolved{first, second}, Executor: executor})
|
||||
if err != nil {
|
||||
t.Fatalf("InspectPromptExecutions() error = %v", err)
|
||||
}
|
||||
if len(results) != 2 || len(executor.profileRequests) != 1 {
|
||||
t.Fatalf("results/profile requests = %#v/%#v, want two results and one profile inspection", results, executor.profileRequests)
|
||||
}
|
||||
}
|
||||
|
||||
type inspectionPromptRequest struct {
|
||||
id string
|
||||
version string
|
||||
@@ -114,6 +138,7 @@ type inspectionPromptRequest struct {
|
||||
|
||||
type inspectionExecutor struct {
|
||||
prompt promptexec.PromptInspection
|
||||
prompts map[string]promptexec.PromptInspection
|
||||
profiles map[string]promptexec.ProfileInspection
|
||||
promptErr error
|
||||
promptRequests []inspectionPromptRequest
|
||||
@@ -125,6 +150,9 @@ func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version
|
||||
if e.promptErr != nil {
|
||||
return promptexec.PromptInspection{}, e.promptErr
|
||||
}
|
||||
if prompt, ok := e.prompts[id]; ok {
|
||||
return prompt, nil
|
||||
}
|
||||
return e.prompt, nil
|
||||
}
|
||||
|
||||
|
||||
21
internal/app/test_helpers_test.go
Normal file
21
internal/app/test_helpers_test.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func mustParse(value string) time.Time {
|
||||
parsed, err := time.Parse(time.RFC3339, value)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func requireNoError(t *testing.T, err error) {
|
||||
t.Helper()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -87,13 +87,7 @@ func newGenerateSummary(result *app.ReportResult, err error) generateSummary {
|
||||
summary.MetadataPath = result.MetadataPath
|
||||
summary.DataPackagePath = result.DataPackagePath
|
||||
summary.PreparationPath = result.PreparationPath
|
||||
if summary.PreparationPath == "" {
|
||||
summary.PreparationPath = result.PreflightPath
|
||||
}
|
||||
summary.ExecutionPath = result.ExecutionPath
|
||||
if summary.ExecutionPath == "" {
|
||||
summary.ExecutionPath = result.Metadata.GeneratedTextResultPath
|
||||
}
|
||||
summary.LLMDebugPath = result.LLMDebugPath
|
||||
summary.GeneratedTextRawPath = result.GeneratedTextRawPath
|
||||
summary.GeneratedTextPath = result.GeneratedTextPath
|
||||
|
||||
@@ -20,8 +20,8 @@ Usage:
|
||||
weatherreporter generate today [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--date YYYY-MM-DD] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter generate tomorrow [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter generate hourly [--config PATH] [--units VALUE] [--tz NAME] [--out PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
|
||||
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--quiet]
|
||||
weatherreporter run morning [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter run evening [--config PATH] [--units VALUE] [--tz NAME] [--out-dir PATH] [--llm-debug-dir PATH] [--quiet]
|
||||
weatherreporter inspect reports [--config PATH] [--limit N]
|
||||
weatherreporter inspect metadata [--config PATH] RUN_ID
|
||||
weatherreporter inspect modules [--config PATH] RUN_ID
|
||||
@@ -98,18 +98,18 @@ func (r Runner) Run(ctx context.Context, args []string, stdout io.Writer, stderr
|
||||
}
|
||||
|
||||
type commonOptions struct {
|
||||
ConfigPath string
|
||||
Units string
|
||||
Timezone string
|
||||
Output string
|
||||
OutputDir string
|
||||
Quiet bool
|
||||
ConfigPath string
|
||||
Units string
|
||||
Timezone string
|
||||
Output string
|
||||
OutputDir string
|
||||
LLMDebugDir string
|
||||
Quiet bool
|
||||
}
|
||||
|
||||
type generateOptions struct {
|
||||
commonOptions
|
||||
Date string
|
||||
LLMDebugDir string
|
||||
Date string
|
||||
}
|
||||
|
||||
type inspectOptions struct {
|
||||
@@ -285,7 +285,11 @@ func (r Runner) resolveRunAction(args []string) (app.BatchRequest, commonOptions
|
||||
if err != nil {
|
||||
return app.BatchRequest{}, commonOptions{}, err
|
||||
}
|
||||
return app.BatchRequest{Config: cfg, Batch: batch, Now: r.Clock.Now(), OutputDir: opts.OutputDir}, opts, nil
|
||||
executor, err := r.promptExecutor(cfg.Promptkit)
|
||||
if err != nil {
|
||||
return app.BatchRequest{}, commonOptions{}, err
|
||||
}
|
||||
return app.BatchRequest{Config: cfg, Batch: batch, Now: r.Clock.Now(), OutputDir: opts.OutputDir, LLMDebugDir: opts.LLMDebugDir, Executor: executor}, opts, nil
|
||||
}
|
||||
|
||||
func resolveRun(args []string) (app.BatchRequest, error) {
|
||||
@@ -298,7 +302,6 @@ func parseGenerateFlags(report app.ReportKind, args []string) (generateOptions,
|
||||
opts := generateOptions{}
|
||||
addCommonFlags(fs, &opts.commonOptions, true)
|
||||
fs.BoolVar(&opts.Quiet, "quiet", false, "suppress successful action output")
|
||||
fs.StringVar(&opts.LLMDebugDir, "llm-debug-dir", "", "write sensitive prompt debug artifacts under PATH")
|
||||
if report == app.ReportDaily || report == app.ReportToday {
|
||||
fs.StringVar(&opts.Date, "date", "", "report date in YYYY-MM-DD")
|
||||
}
|
||||
@@ -364,6 +367,7 @@ func addCommonFlags(fs *flag.FlagSet, opts *commonOptions, includeOutput bool) {
|
||||
fs.StringVar(&opts.ConfigPath, "config", "", "configuration file path")
|
||||
fs.StringVar(&opts.Units, "units", "", "weather API units")
|
||||
fs.StringVar(&opts.Timezone, "tz", "", "weather API timezone")
|
||||
fs.StringVar(&opts.LLMDebugDir, "llm-debug-dir", "", "write sensitive prompt debug artifacts under PATH")
|
||||
if includeOutput {
|
||||
fs.StringVar(&opts.Output, "out", "", "extra Markdown report copy path")
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
37
internal/cli/run_test.go
Normal file
37
internal/cli/run_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
)
|
||||
|
||||
func TestParseRunFlagsAcceptsPromptDebugDirectory(t *testing.T) {
|
||||
opts, err := parseRunFlags([]string{"--llm-debug-dir", "/tmp/prompt-debug"})
|
||||
if err != nil || opts.LLMDebugDir != "/tmp/prompt-debug" {
|
||||
t.Fatalf("parseRunFlags() = %#v, %v", opts, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveRunActionConstructsOneExecutor(t *testing.T) {
|
||||
configPath := filepath.Join(t.TempDir(), "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte("workspace:\n root: "+filepath.Join(t.TempDir(), "workspace")+"\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
calls := 0
|
||||
runner := Runner{
|
||||
Clock: timeutil.FixedClock{Time: time.Date(2026, 5, 29, 8, 0, 0, 0, time.UTC)},
|
||||
ExecutorFactory: func(PromptExecutorConfig) (promptexec.Executor, error) {
|
||||
calls++
|
||||
return factoryExecutor{}, nil
|
||||
},
|
||||
}
|
||||
req, _, err := runner.resolveRunAction([]string{"morning", "--config", configPath, "--llm-debug-dir", "/tmp/debug"})
|
||||
if err != nil || calls != 1 || req.Executor == nil || req.LLMDebugDir != "/tmp/debug" {
|
||||
t.Fatalf("resolveRunAction() request/error/calls = %#v/%v/%d", req, err, calls)
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,6 @@ type Config struct {
|
||||
Secrets SecretsConfig `yaml:"secrets"`
|
||||
Notify NotifyConfig `yaml:"notify"`
|
||||
MissingSource MissingSourceConfig `yaml:"missing_source"`
|
||||
Scriptorium ScriptoriumConfig `yaml:"scriptorium"`
|
||||
Promptkit PromptkitConfig `yaml:"promptkit"`
|
||||
Workspace WorkspaceConfig `yaml:"workspace"`
|
||||
Dayparts []DaypartConfig `yaml:"dayparts"`
|
||||
@@ -82,14 +81,6 @@ type MissingSourceConfig struct {
|
||||
Sources map[string]MissingSourcePolicy `yaml:"sources"`
|
||||
}
|
||||
|
||||
type ScriptoriumConfig struct {
|
||||
Binary string `yaml:"binary"`
|
||||
ConfigPath string `yaml:"config_path"`
|
||||
Profile string `yaml:"profile"`
|
||||
Timeout time.Duration `yaml:"timeout"`
|
||||
ExtraArgs []string `yaml:"extra_args"`
|
||||
}
|
||||
|
||||
type PromptkitConfig struct {
|
||||
Profile string `yaml:"profile"`
|
||||
ProfileFile string `yaml:"profile_file"`
|
||||
|
||||
@@ -144,8 +144,8 @@ func TestLoadMinimalExampleConfig(t *testing.T) {
|
||||
if cfg.WeatherAPI.Units != "us" {
|
||||
t.Fatalf("Units = %q, want default us", cfg.WeatherAPI.Units)
|
||||
}
|
||||
if cfg.Scriptorium.Binary != "scriptorium" {
|
||||
t.Fatalf("Scriptorium.Binary = %q, want default scriptorium", cfg.Scriptorium.Binary)
|
||||
if cfg.Promptkit.Timeout != 2*time.Minute || cfg.Promptkit.Local.ConcurrencyLimit != 1 {
|
||||
t.Fatalf("Promptkit defaults = %#v", cfg.Promptkit)
|
||||
}
|
||||
if cfg.Workspace.Root != "workspace" {
|
||||
t.Fatalf("Workspace.Root = %q, want default workspace", cfg.Workspace.Root)
|
||||
@@ -158,6 +158,13 @@ func TestLoadMinimalExampleConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsRetiredExecutionConfiguration(t *testing.T) {
|
||||
_, err := LoadFile(writeConfig(t, "scriptorium:\n binary: scriptorium\n"))
|
||||
if err == nil || !strings.Contains(err.Error(), "migrate to promptkit") {
|
||||
t.Fatalf("LoadFile() error = %v, want actionable migration error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadReportModuleOverrides(t *testing.T) {
|
||||
path := writeConfig(t, `
|
||||
reports:
|
||||
|
||||
@@ -43,10 +43,6 @@ func Defaults() Config {
|
||||
Default: MissingSourceWarn,
|
||||
Sources: map[string]MissingSourcePolicy{},
|
||||
},
|
||||
Scriptorium: ScriptoriumConfig{
|
||||
Binary: "scriptorium",
|
||||
Timeout: 2 * time.Minute,
|
||||
},
|
||||
Promptkit: PromptkitConfig{
|
||||
Timeout: 2 * time.Minute,
|
||||
Local: PromptkitLocalConfig{
|
||||
|
||||
@@ -59,6 +59,9 @@ func mergeFile(cfg *Config, path string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("read config %q: %w", path, err)
|
||||
}
|
||||
if err := rejectRetiredExecutionConfig(data); err != nil {
|
||||
return fmt.Errorf("parse config %q: %w", path, err)
|
||||
}
|
||||
if err := yaml.Unmarshal(data, cfg); err != nil {
|
||||
return fmt.Errorf("parse config %q: %w", path, err)
|
||||
}
|
||||
@@ -70,3 +73,20 @@ func mergeFile(cfg *Config, path string) error {
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func rejectRetiredExecutionConfig(data []byte) error {
|
||||
var document yaml.Node
|
||||
if err := yaml.Unmarshal(data, &document); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(document.Content) == 0 || document.Content[0].Kind != yaml.MappingNode {
|
||||
return nil
|
||||
}
|
||||
root := document.Content[0]
|
||||
for i := 0; i+1 < len(root.Content); i += 2 {
|
||||
if root.Content[i].Value == "scriptorium" {
|
||||
return fmt.Errorf("scriptorium configuration is no longer supported; migrate to promptkit configuration")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -59,12 +59,6 @@ func Validate(cfg Config) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if cfg.Scriptorium.Binary == "" {
|
||||
return fmt.Errorf("scriptorium.binary is required")
|
||||
}
|
||||
if cfg.Scriptorium.Timeout <= 0 {
|
||||
return fmt.Errorf("scriptorium.timeout must be greater than zero")
|
||||
}
|
||||
if err := validatePromptkit(cfg.Promptkit); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -27,22 +27,16 @@ type FilesystemStore struct {
|
||||
}
|
||||
|
||||
type ArtifactPaths struct {
|
||||
ModuleSnapshot string `json:"moduleSnapshot"`
|
||||
Metadata string `json:"metadata"`
|
||||
DataPackage string `json:"dataPackage"`
|
||||
Preparation string `json:"preparation,omitempty"`
|
||||
Execution string `json:"execution,omitempty"`
|
||||
// Preflight is retained for the temporary Scriptorium write path. Remove it
|
||||
// with that integration's cutover.
|
||||
Preflight string `json:"preflight"`
|
||||
ModuleSnapshot string `json:"moduleSnapshot"`
|
||||
Metadata string `json:"metadata"`
|
||||
DataPackage string `json:"dataPackage"`
|
||||
Preparation string `json:"preparation,omitempty"`
|
||||
Execution string `json:"execution,omitempty"`
|
||||
Notification string `json:"notification,omitempty"`
|
||||
RenderedReport string `json:"renderedReport,omitempty"`
|
||||
GeneratedTextRaw string `json:"generatedTextRaw,omitempty"`
|
||||
// GeneratedTextResult is retained for the temporary Scriptorium write path.
|
||||
// Remove it with that integration's cutover.
|
||||
GeneratedTextResult string `json:"generatedTextResult,omitempty"`
|
||||
GeneratedText string `json:"generatedText,omitempty"`
|
||||
RenderContext string `json:"renderContext,omitempty"`
|
||||
GeneratedText string `json:"generatedText,omitempty"`
|
||||
RenderContext string `json:"renderContext,omitempty"`
|
||||
}
|
||||
|
||||
type ReportRecord struct {
|
||||
@@ -98,18 +92,16 @@ func (s *FilesystemStore) Paths(resolved report.Resolved) (ArtifactPaths, error)
|
||||
}
|
||||
validDate := resolved.ValidPeriod.Start.Format("2006-01-02")
|
||||
return ArtifactPaths{
|
||||
ModuleSnapshot: s.join(s.snapshotsDir, group, validDate, "modules."+metadata.RunID+".json"),
|
||||
Metadata: s.join(s.snapshotsDir, group, validDate, "metadata."+metadata.RunID+".json"),
|
||||
DataPackage: s.join(s.dataPackagesDir, group, validDate, "data_package."+metadata.RunID+".yaml"),
|
||||
Preparation: s.join(s.preflightDir, group, validDate, "prompt_preparation."+metadata.RunID+".json"),
|
||||
Execution: s.join(s.snapshotsDir, group, validDate, "prompt_execution."+metadata.RunID+".json"),
|
||||
Preflight: s.join(s.preflightDir, group, validDate, "render."+metadata.RunID+".json"),
|
||||
Notification: s.join(s.notificationsDir, group, validDate, "distributor."+metadata.RunID+".json"),
|
||||
RenderedReport: s.join(s.reportsDir, group, validDate, "report."+metadata.RunID+".md"),
|
||||
GeneratedTextRaw: s.join(s.snapshotsDir, group, validDate, "generated_text_raw."+metadata.RunID+".json"),
|
||||
GeneratedTextResult: s.join(s.snapshotsDir, group, validDate, "generated_text_result."+metadata.RunID+".json"),
|
||||
GeneratedText: s.join(s.snapshotsDir, group, validDate, "generated_text."+metadata.RunID+".json"),
|
||||
RenderContext: s.join(s.snapshotsDir, group, validDate, "render_context."+metadata.RunID+".json"),
|
||||
ModuleSnapshot: s.join(s.snapshotsDir, group, validDate, "modules."+metadata.RunID+".json"),
|
||||
Metadata: s.join(s.snapshotsDir, group, validDate, "metadata."+metadata.RunID+".json"),
|
||||
DataPackage: s.join(s.dataPackagesDir, group, validDate, "data_package."+metadata.RunID+".yaml"),
|
||||
Preparation: s.join(s.preflightDir, group, validDate, "prompt_preparation."+metadata.RunID+".json"),
|
||||
Execution: s.join(s.snapshotsDir, group, validDate, "prompt_execution."+metadata.RunID+".json"),
|
||||
Notification: s.join(s.notificationsDir, group, validDate, "distributor."+metadata.RunID+".json"),
|
||||
RenderedReport: s.join(s.reportsDir, group, validDate, "report."+metadata.RunID+".md"),
|
||||
GeneratedTextRaw: s.join(s.snapshotsDir, group, validDate, "generated_text_raw."+metadata.RunID+".json"),
|
||||
GeneratedText: s.join(s.snapshotsDir, group, validDate, "generated_text."+metadata.RunID+".json"),
|
||||
RenderContext: s.join(s.snapshotsDir, group, validDate, "render_context."+metadata.RunID+".json"),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -141,12 +133,6 @@ func (s *FilesystemStore) SaveDataPackageBytes(_ context.Context, resolved repor
|
||||
return paths.DataPackage, nil
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) SavePreflight(_ context.Context, resolved report.Resolved, artifact PreflightArtifact) (string, error) {
|
||||
return s.saveResolvedJSON(resolved, func(paths ArtifactPaths) string {
|
||||
return paths.Preflight
|
||||
}, artifact)
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) SavePromptPreparation(_ context.Context, resolved report.Resolved, artifact PromptPreparationArtifact) (string, error) {
|
||||
if artifact.SchemaVersion == "" {
|
||||
artifact.SchemaVersion = PromptPreparationSchemaVersion
|
||||
@@ -213,12 +199,6 @@ func (s *FilesystemStore) SaveGeneratedTextRaw(_ context.Context, resolved repor
|
||||
}, data)
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) SaveGeneratedTextResult(_ context.Context, resolved report.Resolved, value any) (string, error) {
|
||||
return s.saveResolvedJSON(resolved, func(paths ArtifactPaths) string {
|
||||
return paths.GeneratedTextResult
|
||||
}, value)
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) SaveGeneratedText(_ context.Context, resolved report.Resolved, data []byte) (string, error) {
|
||||
return s.saveResolvedBytes(resolved, func(paths ArtifactPaths) string {
|
||||
return paths.GeneratedText
|
||||
@@ -431,16 +411,6 @@ func (s *FilesystemStore) LoadGeneratedText(_ context.Context, path string) ([]b
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) LoadGeneratedTextResult(_ context.Context, path string, target any) error {
|
||||
if path == "" {
|
||||
return fmt.Errorf("generated text result path is required")
|
||||
}
|
||||
if target == nil {
|
||||
return fmt.Errorf("generated text result target is required")
|
||||
}
|
||||
return readJSON(path, target)
|
||||
}
|
||||
|
||||
func (s *FilesystemStore) LoadPromptPreparation(_ context.Context, path string) (PromptPreparationArtifact, error) {
|
||||
if path == "" {
|
||||
return PromptPreparationArtifact{}, fmt.Errorf("prompt preparation path is required")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -46,8 +46,8 @@ type Metadata struct {
|
||||
GeneratedTextPath string `json:"generatedTextPath,omitempty"`
|
||||
RenderContextPath string `json:"renderContextPath,omitempty"`
|
||||
|
||||
// These paths are retained only to preserve records written by the temporary
|
||||
// Scriptorium flow. They are never emitted in V2 metadata.
|
||||
// These paths are retained only to read legacy V1 records. They are never
|
||||
// emitted in V2 metadata.
|
||||
PreflightPath string `json:"-"`
|
||||
GeneratedTextResultPath string `json:"-"`
|
||||
}
|
||||
@@ -170,47 +170,22 @@ func (m Metadata) Validate() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func BuildMetadataFromBriefingMetadata(resolved report.Resolved, briefingMetadata briefing.Metadata, paths ArtifactPaths) Metadata {
|
||||
metadata := resolved.Metadata()
|
||||
out := Metadata{
|
||||
SchemaVersion: MetadataSchemaVersionV1,
|
||||
RunID: metadata.RunID,
|
||||
MetadataPath: paths.Metadata,
|
||||
ReportID: metadata.ReportID,
|
||||
Variant: briefingMetadata.Variant,
|
||||
PromptID: metadata.PromptID,
|
||||
GeneratedAt: metadata.GeneratedAt,
|
||||
Timezone: metadata.Timezone,
|
||||
ValidPeriod: metadata.ValidPeriod,
|
||||
Location: copyLocation(briefingMetadata.Location),
|
||||
SourceLocationID: briefingMetadata.SourceLocationID,
|
||||
SourceLocation: briefingMetadata.SourceLocation,
|
||||
Sources: briefingMetadata.Sources,
|
||||
SourceWarnings: briefingMetadata.SourceWarnings,
|
||||
ModuleSnapshotPath: paths.ModuleSnapshot,
|
||||
DataPackagePath: paths.DataPackage,
|
||||
PreflightPath: paths.Preflight,
|
||||
RenderedReportPath: paths.RenderedReport,
|
||||
}
|
||||
out.GeneratedTextSchemaID = resolved.Definition.GeneratedTextSchemaID
|
||||
out.GeneratedTextRawPath = paths.GeneratedTextRaw
|
||||
out.GeneratedTextResultPath = paths.GeneratedTextResult
|
||||
out.GeneratedTextPath = paths.GeneratedText
|
||||
out.RenderContextPath = paths.RenderContext
|
||||
return out
|
||||
}
|
||||
|
||||
// BuildPromptMetadataFromBriefingMetadata creates the V2 record used by the
|
||||
// prompt execution workflow. Callers populate preparation and execution paths
|
||||
// only after their corresponding artifacts have been saved.
|
||||
func BuildPromptMetadataFromBriefingMetadata(resolved report.Resolved, briefingMetadata briefing.Metadata, paths ArtifactPaths) Metadata {
|
||||
legacy := BuildMetadataFromBriefingMetadata(resolved, briefingMetadata, paths)
|
||||
legacy.SchemaVersion = MetadataSchemaVersion
|
||||
legacy.PreparationPath = ""
|
||||
legacy.ExecutionPath = ""
|
||||
legacy.PreflightPath = ""
|
||||
legacy.GeneratedTextResultPath = ""
|
||||
return legacy
|
||||
metadata := resolved.Metadata()
|
||||
return Metadata{
|
||||
SchemaVersion: MetadataSchemaVersion, RunID: metadata.RunID, MetadataPath: paths.Metadata,
|
||||
ReportID: metadata.ReportID, Variant: briefingMetadata.Variant, PromptID: metadata.PromptID,
|
||||
GeneratedAt: metadata.GeneratedAt, Timezone: metadata.Timezone, ValidPeriod: metadata.ValidPeriod,
|
||||
Location: copyLocation(briefingMetadata.Location), SourceLocationID: briefingMetadata.SourceLocationID,
|
||||
SourceLocation: briefingMetadata.SourceLocation, Sources: briefingMetadata.Sources,
|
||||
SourceWarnings: briefingMetadata.SourceWarnings, ModuleSnapshotPath: paths.ModuleSnapshot,
|
||||
DataPackagePath: paths.DataPackage, RenderedReportPath: paths.RenderedReport,
|
||||
GeneratedTextSchemaID: resolved.Definition.GeneratedTextSchemaID, GeneratedTextRawPath: paths.GeneratedTextRaw,
|
||||
GeneratedTextPath: paths.GeneratedText, RenderContextPath: paths.RenderContext,
|
||||
}
|
||||
}
|
||||
|
||||
func copyLocation(location *briefing.LocationContext) *briefing.LocationContext {
|
||||
|
||||
@@ -16,13 +16,11 @@ type Store interface {
|
||||
SaveModuleSnapshot(context.Context, report.Resolved, module.Snapshot) (string, error)
|
||||
SaveDataPackage(context.Context, report.Resolved, promptinput.Package) (string, error)
|
||||
SaveDataPackageBytes(context.Context, report.Resolved, []byte) (string, error)
|
||||
SavePreflight(context.Context, report.Resolved, PreflightArtifact) (string, error)
|
||||
SavePromptPreparation(context.Context, report.Resolved, PromptPreparationArtifact) (string, error)
|
||||
SavePromptExecution(context.Context, report.Resolved, PromptExecutionArtifact) (string, error)
|
||||
SaveDistributorNotification(context.Context, report.Resolved, DistributorNotificationArtifact) (string, error)
|
||||
SaveBatchDistributorNotification(context.Context, BatchDistributorNotificationRef, BatchDistributorNotificationArtifact) (string, error)
|
||||
SaveGeneratedTextRaw(context.Context, report.Resolved, []byte) (string, error)
|
||||
SaveGeneratedTextResult(context.Context, report.Resolved, any) (string, error)
|
||||
SaveGeneratedText(context.Context, report.Resolved, []byte) (string, error)
|
||||
SaveRenderContext(context.Context, report.Resolved, any) (string, error)
|
||||
PrepareRenderedReport(context.Context, report.Resolved) (string, error)
|
||||
@@ -30,7 +28,6 @@ type Store interface {
|
||||
FindPriorSnapshot(context.Context, report.Resolved) (*PriorSnapshot, error)
|
||||
LoadModuleSnapshot(context.Context, string) (module.Snapshot, error)
|
||||
LoadGeneratedText(context.Context, string) ([]byte, error)
|
||||
LoadGeneratedTextResult(context.Context, string, any) error
|
||||
LoadPromptPreparation(context.Context, string) (PromptPreparationArtifact, error)
|
||||
LoadPromptExecution(context.Context, string) (PromptExecutionArtifact, error)
|
||||
LoadRenderContext(context.Context, string, any) error
|
||||
@@ -41,15 +38,6 @@ type PriorSnapshot struct {
|
||||
ModuleSnapshotPath string
|
||||
}
|
||||
|
||||
type PreflightArtifact struct {
|
||||
Command []string `json:"command"`
|
||||
Stdout string `json:"stdout"`
|
||||
Stderr string `json:"stderr"`
|
||||
StdoutTruncated bool `json:"stdoutTruncated,omitempty"`
|
||||
StderrTruncated bool `json:"stderrTruncated,omitempty"`
|
||||
ExitCode int `json:"exitCode"`
|
||||
}
|
||||
|
||||
const DistributorNotificationSchemaVersion = "weatherreporter.distributor_notification.v1"
|
||||
const BatchDistributorNotificationSchemaVersion = "weatherreporter.batch_distributor_notification.v1"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user