177 lines
8.5 KiB
Go
177 lines
8.5 KiB
Go
package app
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/collect"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptdebug"
|
|
"gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec"
|
|
)
|
|
|
|
func TestExecutePreparedProfileRendersWithoutPublishing(t *testing.T) {
|
|
prepared, inspection := preparedDailyProfile(t)
|
|
executor := &generationExecutor{}
|
|
outputPath := filepath.Join(t.TempDir(), "report.md")
|
|
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
|
|
Prepared: prepared, Prompt: inspection,
|
|
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
|
|
Executor: executor,
|
|
})
|
|
if err != nil {
|
|
t.Fatalf("executePreparedProfile() error = %v", err)
|
|
}
|
|
if len(rendered) == 0 || outcome.ValidationStatus != promptexec.ValidationPassed || outcome.ProfileID != inspection.ProfileID || executor.executeCalls != 1 {
|
|
t.Fatalf("outcome/rendered/execution calls = %#v/%q/%d", outcome, rendered, executor.executeCalls)
|
|
}
|
|
if _, statErr := os.Stat(outputPath); !os.IsNotExist(statErr) {
|
|
t.Fatalf("execution unexpectedly published %q: %v", outputPath, statErr)
|
|
}
|
|
}
|
|
|
|
func TestExecutePreparedProfileKeepsDebugCallbackFailureLocal(t *testing.T) {
|
|
prepared, inspection := preparedDailyProfile(t)
|
|
debugWriter, err := promptdebug.NewPromptDebugWriter(t.TempDir())
|
|
if errors.Is(err, promptdebug.ErrSecureCaptureUnsupported) {
|
|
t.Skipf("secure prompt debug capture is unavailable: %v", err)
|
|
}
|
|
if err != nil {
|
|
t.Fatalf("NewPromptDebugWriter() error = %v", err)
|
|
}
|
|
executor := &generationExecutor{}
|
|
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{
|
|
Prepared: prepared, Prompt: inspection,
|
|
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
|
|
Executor: executor, DebugWriter: debugWriter,
|
|
DebugRef: &promptdebug.PromptDebugRef{ReportID: inspectionResolved(t).Definition.ID, ValidDate: "2026-05-29", RunID: "invalid/path"},
|
|
})
|
|
var executionErr *profileExecutionError
|
|
if err == nil || !errors.As(err, &executionErr) || !executionErr.callbackFailure || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 || executor.executeCalls != 0 || outcome.LLMDebugPath != "" {
|
|
t.Fatalf("outcome/rendered/error/execution calls = %#v/%q/%v/%d", outcome, rendered, err, executor.executeCalls)
|
|
}
|
|
}
|
|
|
|
func TestExecutePreparedProfileBoundsOversizedExecutorOutput(t *testing.T) {
|
|
prepared, inspection := preparedDailyProfile(t)
|
|
marker := "provider-controlled-marker"
|
|
executor := &generationExecutor{rawOutput: []byte(strings.Repeat("x", generatedtext.MaxGeneratedTextBytes+1) + marker)}
|
|
_, _, err := executePreparedProfile(context.Background(), profileExecutionRequest{
|
|
Prepared: prepared, Prompt: inspection,
|
|
Profile: promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName},
|
|
Executor: executor,
|
|
})
|
|
if err == nil || !strings.Contains(err.Error(), "65536-byte limit") {
|
|
t.Fatalf("executePreparedProfile() error = %v, want bounded raw size error", err)
|
|
}
|
|
if len(err.Error()) > 160 || strings.Contains(err.Error(), marker) {
|
|
t.Fatalf("ordinary error leaked provider content: %q", err)
|
|
}
|
|
}
|
|
|
|
func TestExecutePreparedProfileRejectsInconsistentProvenance(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
mutate func(*preparedReport, *PromptInspectionResult, *promptexec.ProfileInspection, *generationExecutor)
|
|
invoked bool
|
|
}{
|
|
{
|
|
name: "prepared definition", mutate: func(prepared *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, _ *generationExecutor) {
|
|
prepared.resolved.Definition.PromptVersion = "different-version"
|
|
},
|
|
},
|
|
{
|
|
name: "missing callback", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.skipPreparation = true
|
|
}, invoked: true,
|
|
},
|
|
{
|
|
name: "duplicate callback", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.preparationCalls = 2
|
|
},
|
|
},
|
|
{
|
|
name: "callback prompt hash", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.prepare = func(value *promptexec.Preparation) { value.PromptHash = "different-hash" }
|
|
},
|
|
},
|
|
{
|
|
name: "callback output schema", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.prepare = func(value *promptexec.Preparation) { value.Output.SchemaPath = "other.generated_text.schema.json" }
|
|
},
|
|
},
|
|
{
|
|
name: "completed profile", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.complete = func(value *promptexec.Execution) { value.ProfileID = "different-profile" }
|
|
}, invoked: true,
|
|
},
|
|
{
|
|
name: "completed rendered prompt hash", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.complete = func(value *promptexec.Execution) { value.RenderedPromptHash = "different-rendered-hash" }
|
|
}, invoked: true,
|
|
},
|
|
{
|
|
name: "completed input hashes", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.prepare = func(value *promptexec.Preparation) {
|
|
value.InputHashes = map[string]string{"data_package": "prepared-hash"}
|
|
}
|
|
executor.complete = func(value *promptexec.Execution) {
|
|
value.InputHashes = map[string]string{"data_package": "completed-hash"}
|
|
}
|
|
}, invoked: true,
|
|
},
|
|
{
|
|
name: "completed validation mode", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.complete = func(value *promptexec.Execution) { value.Validation.Mode = "other" }
|
|
}, invoked: true,
|
|
},
|
|
{
|
|
name: "completed validation schema", mutate: func(_ *preparedReport, _ *PromptInspectionResult, _ *promptexec.ProfileInspection, executor *generationExecutor) {
|
|
executor.complete = func(value *promptexec.Execution) { value.Validation.SchemaPath = "other.generated_text.schema.json" }
|
|
}, invoked: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
prepared, inspection := preparedDailyProfile(t)
|
|
profile := promptexec.ProfileInspection{ProfileID: inspection.ProfileID, BackendID: inspection.BackendID, ModelName: inspection.ModelName}
|
|
executor := &generationExecutor{}
|
|
tt.mutate(&prepared, &inspection, &profile, executor)
|
|
|
|
outcome, rendered, err := executePreparedProfile(context.Background(), profileExecutionRequest{Prepared: prepared, Prompt: inspection, Profile: profile, Executor: executor})
|
|
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration || len(rendered) != 0 {
|
|
t.Fatalf("outcome/rendered/error = %#v/%q/%v", outcome, rendered, err)
|
|
}
|
|
if outcome.ProfileID != profile.ProfileID || outcome.BackendID != profile.BackendID || outcome.ModelName != profile.ModelName || outcome.ValidationStatus != "" {
|
|
t.Fatalf("outcome retained unverified provenance: %#v", outcome)
|
|
}
|
|
if (executor.executeCalls == 1) != tt.invoked {
|
|
t.Fatalf("executor calls = %d, want invoked=%t", executor.executeCalls, tt.invoked)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) {
|
|
t.Helper()
|
|
cfg := generationConfig()
|
|
resolved, err := ResolveGenerate(GenerateRequest{
|
|
Config: cfg, Report: ReportDaily,
|
|
Date: generationTime("2026-05-29T12:00:00-05:00"), Now: generationTime("2026-05-29T08:30:00-05:00"),
|
|
}, generationTime("2026-05-29T08:30:00-05:00"))
|
|
if err != nil {
|
|
t.Fatalf("ResolveGenerate() error = %v", err)
|
|
}
|
|
bundle := generationBundle(t)
|
|
prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}, handler: preparedHandler(t, resolved)})
|
|
if err != nil {
|
|
t.Fatalf("prepareReport() error = %v", err)
|
|
}
|
|
return prepared, PromptInspectionResult{PromptID: resolved.Definition.PromptID, PromptVersion: resolved.Definition.PromptVersion, PromptHash: generationPromptHash, ProfileID: "fixture", BackendID: "fixture", ModelName: "fixture-model"}
|
|
}
|