Assemble comparison application workflow
This commit is contained in:
214
internal/app/comparison.go
Normal file
214
internal/app/comparison.go
Normal file
@@ -0,0 +1,214 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/report"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
)
|
||||
|
||||
// ComparisonRequest describes one explicit, multi-profile report comparison.
|
||||
// It deliberately does not accept a notifier: comparison publication is local.
|
||||
type ComparisonRequest struct {
|
||||
Config config.Config
|
||||
Report ReportKind
|
||||
ProfileIDs []string
|
||||
WorkingDir string
|
||||
OutputDir string
|
||||
Replace bool
|
||||
LLMDebugDir string
|
||||
Date time.Time
|
||||
Clock timeutil.Clock
|
||||
Collector Collector
|
||||
Executor promptexec.Executor
|
||||
}
|
||||
|
||||
// ComparisonResult records the resolved comparison and profile outcomes.
|
||||
type ComparisonResult struct {
|
||||
ComparisonID string
|
||||
ReportID report.ID
|
||||
ReportName string
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
Timezone string
|
||||
ValidPeriod timeutil.Period
|
||||
OutputDirectory string
|
||||
ManifestPath string
|
||||
DataPackagePath string
|
||||
Total int
|
||||
Succeeded int
|
||||
Failed int
|
||||
Results []ComparisonProfileResult
|
||||
}
|
||||
|
||||
// ComparisonProfileResult records one explicitly selected profile.
|
||||
type ComparisonProfileResult struct {
|
||||
Position int
|
||||
ProfileID string
|
||||
BackendID string
|
||||
ModelName string
|
||||
Status string
|
||||
ValidationStatus promptexec.ValidationStatus
|
||||
ReportPath string
|
||||
LLMDebugPath string
|
||||
Error *comparison.SafeError
|
||||
}
|
||||
|
||||
// CompareDetailed assembles, executes, and atomically publishes a comparison
|
||||
// bundle. Profile failures publish a complete partial bundle; all other
|
||||
// failures leave the destination untouched.
|
||||
func CompareDetailed(ctx context.Context, req ComparisonRequest) (*ComparisonResult, error) {
|
||||
if err := comparison.ValidateProfileIDs(req.ProfileIDs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clock := req.Clock
|
||||
if clock == nil {
|
||||
clock = timeutil.SystemClock{}
|
||||
}
|
||||
now := clock.Now()
|
||||
resolved, err := ResolveGenerate(GenerateRequest{Config: req.Config, Report: req.Report, Date: req.Date}, now)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata := resolved.Metadata()
|
||||
comparisonID, err := comparison.BuildComparisonID(metadata.RunID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build comparison identity: %w", err)
|
||||
}
|
||||
result := initialComparisonResult(req, resolved, comparisonID, now.UTC())
|
||||
|
||||
outputName, err := resolved.OutputName()
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("resolve comparison output name: %w", err)
|
||||
}
|
||||
outputDirectory, err := resolveComparisonOutputDirectory(req.WorkingDir, req.OutputDir, req.Config.Output.Directory, outputName)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
initialPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("preflight comparison destination: %w", err)
|
||||
}
|
||||
result.OutputDirectory = initialPlan.Target
|
||||
|
||||
debugWriter, err := promptdebug.NewPromptDebugWriter(req.LLMDebugDir)
|
||||
if err != nil {
|
||||
return result, promptexec.NewError(promptexec.InvalidConfiguration, "initialize prompt debug", err)
|
||||
}
|
||||
inspection, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{
|
||||
Resolved: resolved, ProfileIDs: req.ProfileIDs, Executor: req.Executor, LookupEnv: os.LookupEnv,
|
||||
})
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.PromptID, result.PromptVersion, result.PromptHash = inspection.PromptID, inspection.PromptVersion, inspection.PromptHash
|
||||
|
||||
collection, err := collectWeather(ctx, req.Config, req.Collector)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection})
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("prepare comparison report: %w", err)
|
||||
}
|
||||
|
||||
executed := executeComparisonProfiles(ctx, comparisonExecutionRequest{
|
||||
Prepared: prepared, Inspection: inspection, ComparisonID: comparisonID, DebugWriter: debugWriter, Executor: req.Executor,
|
||||
})
|
||||
result.FinishedAt = clock.Now().UTC()
|
||||
copyComparisonOutcomes(result, executed.Outcomes, false)
|
||||
if executed.Canceled {
|
||||
return result, fmt.Errorf("comparison execution: %w", ctx.Err())
|
||||
}
|
||||
|
||||
bundle := comparisonBundle(result, prepared.dataPackageCopy(), executed.Outcomes)
|
||||
if err := bundle.Validate(); err != nil {
|
||||
return result, fmt.Errorf("build comparison bundle: %w", err)
|
||||
}
|
||||
publicationPlan, err := comparison.PlanDestination(req.WorkingDir, outputDirectory, req.Replace)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("re-preflight comparison destination: %w", err)
|
||||
}
|
||||
if err := comparison.Publish(ctx, publicationPlan, bundle); err != nil {
|
||||
return result, fmt.Errorf("publish comparison bundle: %w", err)
|
||||
}
|
||||
result.OutputDirectory = publicationPlan.Target
|
||||
result.ManifestPath = filepath.Join(publicationPlan.Target, comparison.ManifestFilename)
|
||||
result.DataPackagePath = filepath.Join(publicationPlan.Target, comparison.DataPackageFilename)
|
||||
copyComparisonOutcomes(result, executed.Outcomes, true)
|
||||
|
||||
if result.Failed > 0 {
|
||||
return result, fmt.Errorf("comparison completed with %d failed profiles", result.Failed)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func initialComparisonResult(req ComparisonRequest, resolved report.Resolved, comparisonID string, startedAt time.Time) *ComparisonResult {
|
||||
metadata := resolved.Metadata()
|
||||
return &ComparisonResult{
|
||||
ComparisonID: comparisonID,
|
||||
ReportID: resolved.Definition.ID,
|
||||
ReportName: resolved.Definition.Name,
|
||||
StartedAt: startedAt,
|
||||
Timezone: req.Config.WeatherAPI.Timezone,
|
||||
ValidPeriod: metadata.ValidPeriod,
|
||||
}
|
||||
}
|
||||
|
||||
func copyComparisonOutcomes(result *ComparisonResult, outcomes []comparisonProfileOutcome, published bool) {
|
||||
result.Results = make([]ComparisonProfileResult, len(outcomes))
|
||||
result.Total, result.Succeeded, result.Failed = len(outcomes), 0, 0
|
||||
for index, outcome := range outcomes {
|
||||
profile := ComparisonProfileResult{
|
||||
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||
Status: outcome.Status, ValidationStatus: outcome.ValidationStatus, LLMDebugPath: outcome.LLMDebugPath, Error: outcome.Error,
|
||||
}
|
||||
if published && outcome.Status == comparison.StatusSucceeded {
|
||||
profile.ReportPath = filepath.Join(result.OutputDirectory, outcome.ReportPath)
|
||||
}
|
||||
result.Results[index] = profile
|
||||
if outcome.Status == comparison.StatusSucceeded {
|
||||
result.Succeeded++
|
||||
} else {
|
||||
result.Failed++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonBundle(result *ComparisonResult, dataPackage []byte, outcomes []comparisonProfileOutcome) comparison.LogicalBundle {
|
||||
manifest := comparison.Manifest{
|
||||
SchemaVersion: comparison.SchemaVersion, ComparisonID: result.ComparisonID,
|
||||
StartedAt: result.StartedAt.UTC(), FinishedAt: result.FinishedAt.UTC(),
|
||||
ReportID: string(result.ReportID), Timezone: result.Timezone,
|
||||
ValidPeriod: comparison.ValidPeriod{Start: result.ValidPeriod.Start, End: result.ValidPeriod.End},
|
||||
PromptID: result.PromptID, PromptVersion: result.PromptVersion, PromptHash: result.PromptHash,
|
||||
DataPackage: comparison.DataPackageReference{Path: comparison.DataPackageFilename, SHA256: comparison.SHA256(dataPackage)},
|
||||
Total: result.Total, Succeeded: result.Succeeded, Failed: result.Failed,
|
||||
Results: make([]comparison.Result, len(outcomes)),
|
||||
}
|
||||
bundle := comparison.LogicalBundle{Manifest: manifest, DataPackage: dataPackage}
|
||||
for index, outcome := range outcomes {
|
||||
manifestResult := comparison.Result{
|
||||
Position: outcome.Position, ProfileID: outcome.ProfileID, BackendID: outcome.BackendID, ModelName: outcome.ModelName,
|
||||
Status: outcome.Status, ValidationStatus: string(outcome.ValidationStatus), Error: outcome.Error,
|
||||
}
|
||||
if outcome.Status == comparison.StatusSucceeded {
|
||||
manifestResult.ReportPath = outcome.ReportPath
|
||||
bundle.Reports = append(bundle.Reports, comparison.BundleReport{Position: outcome.Position, Path: outcome.ReportPath, Markdown: outcome.Markdown})
|
||||
}
|
||||
bundle.Manifest.Results[index] = manifestResult
|
||||
}
|
||||
return bundle
|
||||
}
|
||||
180
internal/app/comparison_test.go
Normal file
180
internal/app/comparison_test.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/comparison"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/config"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
||||
"gitea.maximumdirect.net/eric/weatherreporter/internal/timeutil"
|
||||
)
|
||||
|
||||
func TestCompareDetailedPublishesOneCoherentBundle(t *testing.T) {
|
||||
cfg := comparisonConfig()
|
||||
bundle := generationBundle(t)
|
||||
workingDir := t.TempDir()
|
||||
executor := &generationExecutor{}
|
||||
inspectedBeforeCollection := false
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: &generationCollector{bundle: &bundle, beforeRun: func() {
|
||||
inspectedBeforeCollection = executor.promptInspections == 1 && executor.profileInspections == 2
|
||||
}}, Executor: executor,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("CompareDetailed() error = %v", err)
|
||||
}
|
||||
if result == nil || result.Total != 2 || result.Succeeded != 2 || result.Failed != 0 || executor.promptInspections != 1 || executor.profileInspections != 2 || executor.executeCalls != 2 || !inspectedBeforeCollection || result.ManifestPath == "" || result.DataPackagePath == "" {
|
||||
t.Fatalf("result/executor = %#v/%#v", result, executor)
|
||||
}
|
||||
if result.OutputDirectory != filepath.Dir(result.ManifestPath) || !filepath.IsAbs(result.ManifestPath) || !filepath.IsAbs(result.DataPackagePath) {
|
||||
t.Fatalf("published paths = %#v", result)
|
||||
}
|
||||
for index, profile := range result.Results {
|
||||
if profile.Position != index+1 || profile.Status != comparison.StatusSucceeded || !filepath.IsAbs(profile.ReportPath) || profile.Error != nil {
|
||||
t.Fatalf("profile result = %#v", profile)
|
||||
}
|
||||
}
|
||||
data, readErr := os.ReadFile(result.ManifestPath)
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
var manifest comparison.Manifest
|
||||
if err := json.Unmarshal(data, &manifest); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if manifest.ComparisonID != result.ComparisonID || manifest.Total != result.Total || manifest.Succeeded != result.Succeeded || manifest.DataPackage.SHA256 == "" || len(manifest.Results) != 2 {
|
||||
t.Fatalf("manifest = %#v", manifest)
|
||||
}
|
||||
if manifest.Results[0].ReportPath != filepath.Base(result.Results[0].ReportPath) || manifest.Results[1].ReportPath != filepath.Base(result.Results[1].ReportPath) {
|
||||
t.Fatalf("manifest report paths = %#v", manifest.Results)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedPublishesPartialBundleAndReturnsAggregateError(t *testing.T) {
|
||||
cfg := comparisonConfig()
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{executeErrors: map[string]error{"weather-deep": errors.New("provider detail must not escape")}}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: cfg, Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep", "weather-fallback"},
|
||||
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||
})
|
||||
if err == nil || err.Error() != "comparison completed with 1 failed profiles" || result == nil || result.Total != 3 || result.Succeeded != 2 || result.Failed != 1 {
|
||||
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
failure := result.Results[1]
|
||||
if failure.Status != comparison.StatusFailed || failure.ReportPath != "" || failure.Error == nil || strings.Contains(failure.Error.Message, "provider detail") {
|
||||
t.Fatalf("failure = %#v", failure)
|
||||
}
|
||||
if _, statErr := os.Stat(result.ManifestPath); statErr != nil {
|
||||
t.Fatalf("partial manifest: %v", statErr)
|
||||
}
|
||||
if _, statErr := os.Stat(filepath.Join(result.OutputDirectory, filepath.Base(result.Results[0].ReportPath))); statErr != nil {
|
||||
t.Fatalf("successful partial report: %v", statErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedPreflightsBeforePromptOrCollection(t *testing.T) {
|
||||
invalidDestination := filepath.Join(t.TempDir(), "not-a-directory")
|
||||
if err := os.WriteFile(invalidDestination, []byte("x"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bundle := generationBundle(t)
|
||||
collector := &generationCollector{bundle: &bundle}
|
||||
executor := &generationExecutor{}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: t.TempDir(), OutputDir: invalidDestination, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: collector, Executor: executor,
|
||||
})
|
||||
if err == nil || result == nil || collector.called || executor.promptInspections != 0 || executor.executeCalls != 0 || result.ManifestPath != "" {
|
||||
t.Fatalf("result/error/collector/executor = %#v/%v/%#v/%#v", result, err, collector, executor)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedPublishesManifestWhenEveryProfileFails(t *testing.T) {
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{executeErrors: map[string]error{
|
||||
"weather-light": errors.New("first provider failure"), "weather-deep": errors.New("second provider failure"),
|
||||
}}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: t.TempDir(), Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")},
|
||||
Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||
})
|
||||
if err == nil || err.Error() != "comparison completed with 2 failed profiles" || result == nil || result.Succeeded != 0 || result.Failed != 2 || result.ManifestPath == "" {
|
||||
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
for _, profile := range result.Results {
|
||||
if profile.ReportPath != "" || profile.Error == nil {
|
||||
t.Fatalf("failed profile = %#v", profile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedCancellationPreservesPublishedBundle(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
bundle := generationBundle(t)
|
||||
request := ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDir, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: &generationExecutor{},
|
||||
}
|
||||
previous, err := CompareDetailed(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("initial CompareDetailed() error = %v", err)
|
||||
}
|
||||
before, err := os.ReadFile(previous.ManifestPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
request.Replace = true
|
||||
request.Executor = &generationExecutor{cancelBeforeReturn: cancel}
|
||||
result, err := CompareDetailed(ctx, request)
|
||||
if !errors.Is(err, context.Canceled) || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" {
|
||||
t.Fatalf("canceled CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
after, readErr := os.ReadFile(previous.ManifestPath)
|
||||
if readErr != nil || string(after) != string(before) {
|
||||
t.Fatalf("published manifest changed = %q, error = %v", after, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompareDetailedLeavesExistingBundleWhenPublicationPreflightChanges(t *testing.T) {
|
||||
workingDir := t.TempDir()
|
||||
target := filepath.Join(workingDir, "comparison-output")
|
||||
bundle := generationBundle(t)
|
||||
executor := &generationExecutor{beforeExecute: func(promptexec.ExecuteRequest) {
|
||||
_ = os.WriteFile(target, []byte("changed"), 0o600)
|
||||
}}
|
||||
result, err := CompareDetailed(context.Background(), ComparisonRequest{
|
||||
Config: comparisonConfig(), Report: ReportDaily, ProfileIDs: []string{"weather-light", "weather-deep"},
|
||||
WorkingDir: workingDir, OutputDir: target, Date: generationTime("2026-05-29T12:00:00-05:00"),
|
||||
Clock: timeutil.FixedClock{Time: generationTime("2026-05-29T08:30:00-05:00")}, Collector: &generationCollector{bundle: &bundle}, Executor: executor,
|
||||
})
|
||||
if err == nil || result == nil || result.ManifestPath != "" || result.DataPackagePath != "" || result.Results[0].ReportPath != "" {
|
||||
t.Fatalf("CompareDetailed() result/error = %#v/%v", result, err)
|
||||
}
|
||||
data, readErr := os.ReadFile(target)
|
||||
if readErr != nil || string(data) != "changed" {
|
||||
t.Fatalf("destination = %q, error = %v", data, readErr)
|
||||
}
|
||||
}
|
||||
|
||||
func comparisonConfig() config.Config {
|
||||
cfg := config.Defaults()
|
||||
cfg.WeatherAPI.Timezone, cfg.Location.ID = "America/Chicago", "home"
|
||||
return cfg
|
||||
}
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -37,52 +38,77 @@ type generationExecutor struct {
|
||||
called bool
|
||||
executeCalls int
|
||||
promptInspections int
|
||||
profileInspections int
|
||||
inspectErr error
|
||||
executeErr error
|
||||
executeErrors map[string]error
|
||||
beforeExecute func(promptexec.ExecuteRequest)
|
||||
cancelBeforeReturn context.CancelFunc
|
||||
validation promptexec.ValidationStatus
|
||||
rawOutput []byte
|
||||
failedPrompt string
|
||||
}
|
||||
|
||||
var generationExecutorMu sync.Mutex
|
||||
|
||||
func (e *generationExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
|
||||
generationExecutorMu.Lock()
|
||||
defer generationExecutorMu.Unlock()
|
||||
e.promptInspections++
|
||||
if e.inspectErr != nil {
|
||||
return promptexec.PromptInspection{}, e.inspectErr
|
||||
}
|
||||
definition := generationDefinitionForPrompt(id)
|
||||
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: "prompt-hash", DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
||||
return promptexec.PromptInspection{PromptID: id, PromptVersion: version, PromptHash: generationPromptHash, DefaultProfileID: "fixture", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}}, nil
|
||||
}
|
||||
func (*generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||
func (e *generationExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
|
||||
generationExecutorMu.Lock()
|
||||
defer generationExecutorMu.Unlock()
|
||||
e.profileInspections++
|
||||
return promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}, nil
|
||||
}
|
||||
func (e *generationExecutor) Execute(_ context.Context, req promptexec.ExecuteRequest, callback promptexec.PreparationCallback) (*promptexec.Execution, error) {
|
||||
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
|
||||
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||
if err := callback(promptexec.Preparation{PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp}, nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
generationExecutorMu.Lock()
|
||||
e.called = true
|
||||
e.executeCalls++
|
||||
if e.executeErr != nil {
|
||||
return nil, e.executeErr
|
||||
}
|
||||
beforeExecute := e.beforeExecute
|
||||
profileErr := e.executeErrors[req.ProfileID]
|
||||
executeErr := e.executeErr
|
||||
status := e.validation
|
||||
rawOutput := append([]byte(nil), e.rawOutput...)
|
||||
failedPrompt := e.failedPrompt
|
||||
cancelBeforeReturn := e.cancelBeforeReturn
|
||||
generationExecutorMu.Unlock()
|
||||
if beforeExecute != nil {
|
||||
beforeExecute(req)
|
||||
}
|
||||
if profileErr != nil {
|
||||
return nil, profileErr
|
||||
}
|
||||
if executeErr != nil {
|
||||
return nil, executeErr
|
||||
}
|
||||
if status == "" {
|
||||
status = promptexec.ValidationPassed
|
||||
}
|
||||
if e.failedPrompt == req.PromptID {
|
||||
if failedPrompt == req.PromptID {
|
||||
status = promptexec.ValidationFailed
|
||||
}
|
||||
rawOutput := e.rawOutput
|
||||
if rawOutput == nil {
|
||||
rawOutput = []byte(`{"summary":"Showers are possible during the selected day.","forecast_discussion":["A front will keep rain chances in the forecast."],"precipitation_timing":"Rain is most likely during the afternoon."}`)
|
||||
}
|
||||
if e.cancelBeforeReturn != nil {
|
||||
e.cancelBeforeReturn()
|
||||
if cancelBeforeReturn != nil {
|
||||
cancelBeforeReturn()
|
||||
}
|
||||
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
|
||||
return &promptexec.Execution{RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: generationPromptHash, RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model", StartedAt: stamp, EndedAt: stamp, RawOutput: rawOutput, Validation: promptexec.NewValidation(status, "json_schema", generationDefinitionForPrompt(req.PromptID).GeneratedTextSchemaID+".generated_text.schema.json", nil)}, nil
|
||||
}
|
||||
|
||||
const generationPromptHash = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
|
||||
|
||||
func generationDefinitionForPrompt(promptID string) report.Definition {
|
||||
for _, definition := range report.DefaultRegistry().All() {
|
||||
if definition.PromptID == promptID {
|
||||
|
||||
Reference in New Issue
Block a user